Crawler hinzugefuegt

7 files changed, 276 insertions, 0 deletions

diff --git a/tools/charcrawler/.gitignore b/tools/charcrawler/.gitignore
new file mode 100644
index 0000000..02a07a1
--- /dev/null
+++ b/tools/charcrawler/.gitignore
@@ -0,0 +1,2 @@
+data/
+*.txt
diff --git a/tools/charcrawler/crawler/acd.go b/tools/charcrawler/crawler/acd.go
new file mode 100644
index 0000000..31efb0d
--- /dev/null
+++ b/tools/charcrawler/crawler/acd.go
@@ -0,0 +1,81 @@
+package crawler
+
+import (
+        "log"
+        "strconv"
+        "strings"
+
+        "github.com/PuerkitoBio/goquery"
+)
+
+type ACDCrawler struct{}
+
+func (a ACDCrawler) Name() string {
+        return "ACD"
+}
+
+func (a ACDCrawler) Crawl(id int) (CharacterData, error) {
+        c := make(CharacterData)
+        doc, err := goquery.NewDocument("http://www.animecharactersdatabase.com/character.php?id=" + strconv.Itoa(id))
+        if err != nil {
+                log.Println(err)
+                return nil, CrawlError
+        }
+        text := doc.Text()
+        if strings.Contains(text, "bad character : try") {
+                return nil, CharacterNotFound
+        }
+        dataTr := doc.Find("#besttable").Next().ChildrenFiltered("tbody").ChildrenFiltered("tr")
+        leftSide := dataTr.Children().Eq(0)
+        rightSide := dataTr.Children().Eq(1)
+        imageCols := rightSide.ChildrenFiltered("table").First().Find("tbody > tr > td")
+        val, _ := imageCols.Eq(0).Find("img").Attr("src")
+        c["__thumb"] = val
+        val, _ = imageCols.Eq(1).Find("a").Attr("href")
+        c["__img"] = val
+        leftSide.ChildrenFiltered("table").Eq(1).Find("tr").Each(func(i int, s *goquery.Selection) {
+                c[s.Find("th").Text()] = s.Find("td").Text()
+        })
+        var key string
+        leftSide.
+                ChildrenFiltered("table").
+                Eq(0).
+                Find("td").
+                Eq(1).
+                Find("dl").
+                Children().
+                Each(func(i int, s *goquery.Selection) {
+                switch goquery.NodeName(s) {
+                case "dt":
+                        key = s.Text()
+                case "dd":
+                        c[key] = s.Text()
+                }
+        })
+        tags := make([]string, 0)
+        leftSide.ChildrenFiltered("div").Eq(0).Find("a").Each(func(i int, s *goquery.Selection) {
+                tags = append(tags, s.Text())
+        })
+        c["__tags"] = tags
+        vas := make([]string, 0)
+        leftSide.ChildrenFiltered("div").Eq(1).Find("a").Each(func(i int, s *goquery.Selection) {
+                vas = append(vas, s.Text())
+        })
+        c["__vas"] = vas
+        leftSide.ChildrenFiltered("dl").Children().Each(func(i int, s *goquery.Selection) {
+                switch goquery.NodeName(s) {
+                case "dt":
+                        key = s.Text()
+                case "dd":
+                        c[key] = s.Text()
+                }
+        })
+        apps := make([]int, 0)
+        rightSide.Find(".tile3top").Each(func(i int, s *goquery.Selection) {
+                val, _ = s.Find("a").Attr("href")
+                id, _ := strconv.Atoi(strings.Split(val, "=")[1])
+                apps = append(apps, id)
+        })
+        c["__appearances"] = apps
+        return c, nil
+}
diff --git a/tools/charcrawler/crawler/acd_anime.go b/tools/charcrawler/crawler/acd_anime.go
new file mode 100644
index 0000000..106655d
--- /dev/null
+++ b/tools/charcrawler/crawler/acd_anime.go
@@ -0,0 +1,41 @@
+package crawler
+
+import (
+        "log"
+        "strconv"
+        "strings"
+
+        "github.com/PuerkitoBio/goquery"
+)
+
+type ACDAnimeCrawler struct{}
+
+func (a ACDAnimeCrawler) Name() string {
+        return "ACDAnime"
+}
+
+func (a ACDAnimeCrawler) Crawl(id int) (CharacterData, error) {
+        c := make(CharacterData)
+        doc, err := goquery.NewDocument("http://www.animecharactersdatabase.com/series.php?id=" + strconv.Itoa(id))
+        if err != nil {
+                log.Println(err)
+                return nil, CrawlError
+        }
+        text := doc.Text()
+        if strings.Contains(text, "bad series : try") {
+                return nil, CharacterNotFound
+        }
+        tds := doc.Find("#besttable > table > tbody > tr > td")
+        val, _ := tds.Eq(0).Find("img").Attr("src")
+        c["__thumb"] = val
+        tds.Eq(1).Find("tr").Each(func(i int, s *goquery.Selection) {
+                key := s.Find("th").Text()
+                value := s.Find("td").Text()
+                if key == "Home Page" {
+                        val, _ = s.Find("td > a").Attr("href")
+                        value = val
+                }
+                c[key] = value
+        })
+        return c, nil
+}
diff --git a/tools/charcrawler/crawler/anidb.go b/tools/charcrawler/crawler/anidb.go
new file mode 100644
index 0000000..12bf05b
--- /dev/null
+++ b/tools/charcrawler/crawler/anidb.go
@@ -0,0 +1,69 @@
+package crawler
+
+import (
+        "log"
+        "strconv"
+        "strings"
+
+        "github.com/PuerkitoBio/goquery"
+)
+
+type AniDBCrawler struct{}
+
+func (a AniDBCrawler) Name() string {
+        return "AniDB"
+}
+
+func (a AniDBCrawler) Crawl(id int) (CharacterData, error) {
+        c := make(CharacterData)
+        doc, err := goquery.NewDocument("http://anidb.net/perl-bin/animedb.pl?show=character&charid=" + strconv.Itoa(id))
+        if err != nil {
+                log.Println(err)
+                return nil, CrawlError
+        }
+        siteErr, _ := doc.Find(".error p").Html()
+        if strings.Contains(siteErr, "Unknown character id") {
+                return nil, CharacterNotFound
+        } else if strings.Contains(siteErr, "BANNED") {
+                return nil, CrawlError
+        }
+        first := true
+        links := make([]string, 0)
+        doc.Find(".characters").Each(func(i int, s *goquery.Selection) {
+                if first {
+                        s.Find(".g_definitionlist tr").Each(func(i int, s *goquery.Selection) {
+                                keyHtml, _ := s.Find("th").Html()
+                                valueHtml, _ := s.Find("td").Html()
+                                key := strings.TrimSpace(keyHtml)
+                                value := strings.TrimSpace(valueHtml)
+                                c[key] = value
+                        })
+                        desc, _ := s.Find(".desc").Html()
+                        c["__desc"] = strings.TrimSpace(desc)
+                        imgUrl, _ := s.Find(".image img").Attr("src")
+                        c["__img"] = imgUrl
+                        animes := make([]map[string]string, 0)
+                        s.Find(".animelist tr").Each(func(i int, s *goquery.Selection) {
+                                if s.HasClass("header") {
+                                        return
+                                }
+                                anime := make(map[string]string)
+                                relHtml, _ := s.Find(".relation").Html()
+                                nameHtml, _ := s.Find(".name a").Html()
+                                rel := strings.TrimSpace(relHtml)
+                                name := strings.TrimSpace(nameHtml)
+                                anime["rel"] = rel
+                                anime["name"] = name
+                                animes = append(animes, anime)
+                        })
+                        c["__animes"] = animes
+                } else {
+                        link, _ := s.Find(".mainname a").Attr("href")
+                        links = append(links, strings.Replace(link, "http://anidb.net/ch", "", 1))
+                }
+                first = false
+        })
+        c["__links"] = links
+
+        return c, nil
+}
diff --git a/tools/charcrawler/crawler/crawler.go b/tools/charcrawler/crawler/crawler.go
new file mode 100644
index 0000000..337bc76
--- /dev/null
+++ b/tools/charcrawler/crawler/crawler.go
@@ -0,0 +1,67 @@
+package crawler
+
+import (
+        "encoding/json"
+        "errors"
+        "io/ioutil"
+        "log"
+        "os"
+        "strconv"
+        "strings"
+        "time"
+)
+
+type CharacterData map[string]interface{}
+
+type Crawler interface {
+        Name() string
+        Crawl(id int) (CharacterData, error)
+}
+
+var Instances []Crawler
+
+var (
+        CrawlError        = errors.New("Error while crawling")
+        CharacterNotFound = errors.New("Character not found")
+)
+
+func Start(c Crawler) {
+        name := c.Name()
+        os.MkdirAll("data/"+name, 0755)
+        log.Printf("Starting Crawler %s...", name)
+        ticker := time.NewTicker(time.Second * 1)
+        current := 1
+        if save, err := ioutil.ReadFile(name + ".txt"); err == nil {
+                s := strings.TrimSpace(string(save))
+                if i, err := strconv.Atoi(s); err == nil {
+                        current = i
+                }
+        }
+        faultCounter := 0
+        for range ticker.C {
+                if faultCounter > 100 {
+                        faultCounter = 0
+                        log.Printf("[%s] Exiting after 100 fails", name)
+                        break
+                }
+                log.Printf("[%s] Crawling %d", name, current)
+                char, err := c.Crawl(current)
+                switch err {
+                case CharacterNotFound:
+                        log.Printf("[%s] Char %d not found!", name, current)
+                        faultCounter++
+                case CrawlError:
+                        panic(err)
+                default:
+                        cData, _ := json.Marshal(char)
+                        ioutil.WriteFile("data/"+name+"/"+strconv.Itoa(current)+".json", cData, 0755)
+                }
+
+                current++
+                ioutil.WriteFile(name+".txt", []byte(strconv.Itoa(current)), os.ModePerm)
+        }
+}
+
+func init() {
+        Instances = append(Instances, new(ACDCrawler), new(ACDAnimeCrawler))
+}
diff --git a/tools/charcrawler/main b/tools/charcrawler/main
new file mode 100755
index 0000000..71f9896
--- /dev/null
+++ b/tools/charcrawler/main
diff --git a/tools/charcrawler/main.go b/tools/charcrawler/main.go
new file mode 100644
index 0000000..0c94151
--- /dev/null
+++ b/tools/charcrawler/main.go
@@ -0,0 +1,16 @@
+package main
+
+import (
+        "time"
+
+        "private/charcrawler/crawler"
+)
+
+func main() {
+        for _, c := range crawler.Instances {
+                go crawler.Start(c)
+        }
+        for {
+                time.Sleep(time.Minute)
+        }
+}