From 1400381e9c193eb7f681d95616eed9070edaab56 Mon Sep 17 00:00:00 2001 From: rtz12 Date: Tue, 29 Mar 2016 17:58:55 +0200 Subject: Crawler hinzugefuegt diff --git a/tools/charcrawler/.gitignore b/tools/charcrawler/.gitignore new file mode 100644 index 0000000..02a07a1 --- /dev/null +++ b/tools/charcrawler/.gitignore @@ -0,0 +1,2 @@ +data/ +*.txt diff --git a/tools/charcrawler/crawler/acd.go b/tools/charcrawler/crawler/acd.go new file mode 100644 index 0000000..31efb0d --- /dev/null +++ b/tools/charcrawler/crawler/acd.go @@ -0,0 +1,81 @@ +package crawler + +import ( + "log" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +type ACDCrawler struct{} + +func (a ACDCrawler) Name() string { + return "ACD" +} + +func (a ACDCrawler) Crawl(id int) (CharacterData, error) { + c := make(CharacterData) + doc, err := goquery.NewDocument("http://www.animecharactersdatabase.com/character.php?id=" + strconv.Itoa(id)) + if err != nil { + log.Println(err) + return nil, CrawlError + } + text := doc.Text() + if strings.Contains(text, "bad character : try") { + return nil, CharacterNotFound + } + dataTr := doc.Find("#besttable").Next().ChildrenFiltered("tbody").ChildrenFiltered("tr") + leftSide := dataTr.Children().Eq(0) + rightSide := dataTr.Children().Eq(1) + imageCols := rightSide.ChildrenFiltered("table").First().Find("tbody > tr > td") + val, _ := imageCols.Eq(0).Find("img").Attr("src") + c["__thumb"] = val + val, _ = imageCols.Eq(1).Find("a").Attr("href") + c["__img"] = val + leftSide.ChildrenFiltered("table").Eq(1).Find("tr").Each(func(i int, s *goquery.Selection) { + c[s.Find("th").Text()] = s.Find("td").Text() + }) + var key string + leftSide. + ChildrenFiltered("table"). + Eq(0). + Find("td"). + Eq(1). + Find("dl"). + Children(). + Each(func(i int, s *goquery.Selection) { + switch goquery.NodeName(s) { + case "dt": + key = s.Text() + case "dd": + c[key] = s.Text() + } + }) + tags := make([]string, 0) + leftSide.ChildrenFiltered("div").Eq(0).Find("a").Each(func(i int, s *goquery.Selection) { + tags = append(tags, s.Text()) + }) + c["__tags"] = tags + vas := make([]string, 0) + leftSide.ChildrenFiltered("div").Eq(1).Find("a").Each(func(i int, s *goquery.Selection) { + vas = append(vas, s.Text()) + }) + c["__vas"] = vas + leftSide.ChildrenFiltered("dl").Children().Each(func(i int, s *goquery.Selection) { + switch goquery.NodeName(s) { + case "dt": + key = s.Text() + case "dd": + c[key] = s.Text() + } + }) + apps := make([]int, 0) + rightSide.Find(".tile3top").Each(func(i int, s *goquery.Selection) { + val, _ = s.Find("a").Attr("href") + id, _ := strconv.Atoi(strings.Split(val, "=")[1]) + apps = append(apps, id) + }) + c["__appearances"] = apps + return c, nil +} diff --git a/tools/charcrawler/crawler/acd_anime.go b/tools/charcrawler/crawler/acd_anime.go new file mode 100644 index 0000000..106655d --- /dev/null +++ b/tools/charcrawler/crawler/acd_anime.go @@ -0,0 +1,41 @@ +package crawler + +import ( + "log" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +type ACDAnimeCrawler struct{} + +func (a ACDAnimeCrawler) Name() string { + return "ACDAnime" +} + +func (a ACDAnimeCrawler) Crawl(id int) (CharacterData, error) { + c := make(CharacterData) + doc, err := goquery.NewDocument("http://www.animecharactersdatabase.com/series.php?id=" + strconv.Itoa(id)) + if err != nil { + log.Println(err) + return nil, CrawlError + } + text := doc.Text() + if strings.Contains(text, "bad series : try") { + return nil, CharacterNotFound + } + tds := doc.Find("#besttable > table > tbody > tr > td") + val, _ := tds.Eq(0).Find("img").Attr("src") + c["__thumb"] = val + tds.Eq(1).Find("tr").Each(func(i int, s *goquery.Selection) { + key := s.Find("th").Text() + value := s.Find("td").Text() + if key == "Home Page" { + val, _ = s.Find("td > a").Attr("href") + value = val + } + c[key] = value + }) + return c, nil +} diff --git a/tools/charcrawler/crawler/anidb.go b/tools/charcrawler/crawler/anidb.go new file mode 100644 index 0000000..12bf05b --- /dev/null +++ b/tools/charcrawler/crawler/anidb.go @@ -0,0 +1,69 @@ +package crawler + +import ( + "log" + "strconv" + "strings" + + "github.com/PuerkitoBio/goquery" +) + +type AniDBCrawler struct{} + +func (a AniDBCrawler) Name() string { + return "AniDB" +} + +func (a AniDBCrawler) Crawl(id int) (CharacterData, error) { + c := make(CharacterData) + doc, err := goquery.NewDocument("http://anidb.net/perl-bin/animedb.pl?show=character&charid=" + strconv.Itoa(id)) + if err != nil { + log.Println(err) + return nil, CrawlError + } + siteErr, _ := doc.Find(".error p").Html() + if strings.Contains(siteErr, "Unknown character id") { + return nil, CharacterNotFound + } else if strings.Contains(siteErr, "BANNED") { + return nil, CrawlError + } + first := true + links := make([]string, 0) + doc.Find(".characters").Each(func(i int, s *goquery.Selection) { + if first { + s.Find(".g_definitionlist tr").Each(func(i int, s *goquery.Selection) { + keyHtml, _ := s.Find("th").Html() + valueHtml, _ := s.Find("td").Html() + key := strings.TrimSpace(keyHtml) + value := strings.TrimSpace(valueHtml) + c[key] = value + }) + desc, _ := s.Find(".desc").Html() + c["__desc"] = strings.TrimSpace(desc) + imgUrl, _ := s.Find(".image img").Attr("src") + c["__img"] = imgUrl + animes := make([]map[string]string, 0) + s.Find(".animelist tr").Each(func(i int, s *goquery.Selection) { + if s.HasClass("header") { + return + } + anime := make(map[string]string) + relHtml, _ := s.Find(".relation").Html() + nameHtml, _ := s.Find(".name a").Html() + rel := strings.TrimSpace(relHtml) + name := strings.TrimSpace(nameHtml) + anime["rel"] = rel + anime["name"] = name + animes = append(animes, anime) + }) + c["__animes"] = animes + } else { + link, _ := s.Find(".mainname a").Attr("href") + links = append(links, strings.Replace(link, "http://anidb.net/ch", "", 1)) + } + first = false + }) + c["__links"] = links + + return c, nil +} diff --git a/tools/charcrawler/crawler/crawler.go b/tools/charcrawler/crawler/crawler.go new file mode 100644 index 0000000..337bc76 --- /dev/null +++ b/tools/charcrawler/crawler/crawler.go @@ -0,0 +1,67 @@ +package crawler + +import ( + "encoding/json" + "errors" + "io/ioutil" + "log" + "os" + "strconv" + "strings" + "time" +) + +type CharacterData map[string]interface{} + +type Crawler interface { + Name() string + Crawl(id int) (CharacterData, error) +} + +var Instances []Crawler + +var ( + CrawlError = errors.New("Error while crawling") + CharacterNotFound = errors.New("Character not found") +) + +func Start(c Crawler) { + name := c.Name() + os.MkdirAll("data/"+name, 0755) + log.Printf("Starting Crawler %s...", name) + ticker := time.NewTicker(time.Second * 1) + current := 1 + if save, err := ioutil.ReadFile(name + ".txt"); err == nil { + s := strings.TrimSpace(string(save)) + if i, err := strconv.Atoi(s); err == nil { + current = i + } + } + faultCounter := 0 + for range ticker.C { + if faultCounter > 100 { + faultCounter = 0 + log.Printf("[%s] Exiting after 100 fails", name) + break + } + log.Printf("[%s] Crawling %d", name, current) + char, err := c.Crawl(current) + switch err { + case CharacterNotFound: + log.Printf("[%s] Char %d not found!", name, current) + faultCounter++ + case CrawlError: + panic(err) + default: + cData, _ := json.Marshal(char) + ioutil.WriteFile("data/"+name+"/"+strconv.Itoa(current)+".json", cData, 0755) + } + + current++ + ioutil.WriteFile(name+".txt", []byte(strconv.Itoa(current)), os.ModePerm) + } +} + +func init() { + Instances = append(Instances, new(ACDCrawler), new(ACDAnimeCrawler)) +} diff --git a/tools/charcrawler/main b/tools/charcrawler/main new file mode 100755 index 0000000..71f9896 Binary files /dev/null and b/tools/charcrawler/main differ diff --git a/tools/charcrawler/main.go b/tools/charcrawler/main.go new file mode 100644 index 0000000..0c94151 --- /dev/null +++ b/tools/charcrawler/main.go @@ -0,0 +1,16 @@ +package main + +import ( + "time" + + "private/charcrawler/crawler" +) + +func main() { + for _, c := range crawler.Instances { + go crawler.Start(c) + } + for { + time.Sleep(time.Minute) + } +} -- cgit v0.10.1 From c90b115314981c2452799df80ec9f5b15cb493ff Mon Sep 17 00:00:00 2001 From: rtz12 Date: Tue, 29 Mar 2016 18:04:00 +0200 Subject: Sperrungsdetektion diff --git a/tools/charcrawler/crawler/acd.go b/tools/charcrawler/crawler/acd.go index 31efb0d..9a37b95 100644 --- a/tools/charcrawler/crawler/acd.go +++ b/tools/charcrawler/crawler/acd.go @@ -25,6 +25,9 @@ func (a ACDCrawler) Crawl(id int) (CharacterData, error) { if strings.Contains(text, "bad character : try") { return nil, CharacterNotFound } + if strings.Contains(text, "Your IP has been blocked") { + return nil, Banned + } dataTr := doc.Find("#besttable").Next().ChildrenFiltered("tbody").ChildrenFiltered("tr") leftSide := dataTr.Children().Eq(0) rightSide := dataTr.Children().Eq(1) diff --git a/tools/charcrawler/crawler/acd_anime.go b/tools/charcrawler/crawler/acd_anime.go index 106655d..c697b43 100644 --- a/tools/charcrawler/crawler/acd_anime.go +++ b/tools/charcrawler/crawler/acd_anime.go @@ -25,6 +25,9 @@ func (a ACDAnimeCrawler) Crawl(id int) (CharacterData, error) { if strings.Contains(text, "bad series : try") { return nil, CharacterNotFound } + if strings.Contains(text, "Your IP has been blocked") { + return nil, Banned + } tds := doc.Find("#besttable > table > tbody > tr > td") val, _ := tds.Eq(0).Find("img").Attr("src") c["__thumb"] = val diff --git a/tools/charcrawler/crawler/crawler.go b/tools/charcrawler/crawler/crawler.go index 337bc76..64d1a27 100644 --- a/tools/charcrawler/crawler/crawler.go +++ b/tools/charcrawler/crawler/crawler.go @@ -23,6 +23,7 @@ var Instances []Crawler var ( CrawlError = errors.New("Error while crawling") CharacterNotFound = errors.New("Character not found") + Banned = errors.New("Crawler banned from source") ) func Start(c Crawler) { -- cgit v0.10.1 From ae4f7f89849c4e94b388aabdb28d8f9f110df3c2 Mon Sep 17 00:00:00 2001 From: rtz12 Date: Tue, 29 Mar 2016 18:06:16 +0200 Subject: Mit dem neuen Fehler handeln diff --git a/tools/charcrawler/crawler/crawler.go b/tools/charcrawler/crawler/crawler.go index 64d1a27..3d27ea2 100644 --- a/tools/charcrawler/crawler/crawler.go +++ b/tools/charcrawler/crawler/crawler.go @@ -53,6 +53,8 @@ func Start(c Crawler) { faultCounter++ case CrawlError: panic(err) + case Banned: + panic(err) default: cData, _ := json.Marshal(char) ioutil.WriteFile("data/"+name+"/"+strconv.Itoa(current)+".json", cData, 0755) -- cgit v0.10.1