aboutsummaryrefslogtreecommitdiff
path: root/tools/charcrawler/crawler/acd.go
blob: 14f22a260234cb632b7aa28dc5a07a8dd5ab6280 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
package crawler

import (
	"log"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

type ACDCrawler struct{}

func (a ACDCrawler) Name() string {
	return "ACD"
}

func (a ACDCrawler) Crawl(id int) (CharacterData, error) {
	c := make(CharacterData)
	doc, err := goquery.NewDocument("http://www.animecharactersdatabase.com/character.php?id=" + strconv.Itoa(id))
	if err != nil {
		log.Println(err)
		return nil, CrawlError
	}
	text := doc.Text()
	if strings.Contains(text, "bad character : try") {
		return nil, CharacterNotFound
	}
	if strings.Contains(text, "Your IP has been blocked") {
		return nil, Banned
	}
	dataTr := doc.Find("#profile").ChildrenFiltered("table").Eq(0).ChildrenFiltered("tbody").ChildrenFiltered("tr")
	leftSide := dataTr.Children().Eq(0)
	rightSide := dataTr.Children().Eq(1)
	imageCols := rightSide.ChildrenFiltered("table").First().Find("tbody > tr > td")
	val, _ := imageCols.Eq(0).Find("img").Attr("src")
	c["__thumb"] = val
	val, _ = imageCols.Eq(1).Find("a").Attr("href")
	c["__img"] = val
	leftSide.ChildrenFiltered("table").Eq(1).Find("tr").Each(func(i int, s *goquery.Selection) {
		c[s.Find("th").Text()] = s.Find("td").Text()
	})
	var key string
	leftSide.
		ChildrenFiltered("table").
		Eq(0).
		Find("td").
		Eq(1).
		Find("dl").
		Children().
		Each(func(i int, s *goquery.Selection) {
		switch goquery.NodeName(s) {
		case "dt":
			key = s.Text()
		case "dd":
			c[key] = s.Text()
		}
	})
	tags := make([]string, 0)
	leftSide.ChildrenFiltered("div").Eq(0).Find("a").Each(func(i int, s *goquery.Selection) {
		tags = append(tags, s.Text())
	})
	c["__tags"] = tags
	vas := make([]string, 0)
	leftSide.ChildrenFiltered("div").Eq(1).Find("a").Each(func(i int, s *goquery.Selection) {
		vas = append(vas, s.Text())
	})
	c["__vas"] = vas
	leftSide.ChildrenFiltered("dl").Children().Each(func(i int, s *goquery.Selection) {
		switch goquery.NodeName(s) {
		case "dt":
			key = s.Text()
		case "dd":
			c[key] = s.Text()
		}
	})
	apps := make([]int, 0)
	rightSide.Find(".tile3top").Each(func(i int, s *goquery.Selection) {
		val, _ = s.Find("a").Attr("href")
		id, _ := strconv.Atoi(strings.Split(val, "=")[1])
		apps = append(apps, id)
	})
	c["__appearances"] = apps
	return c, nil
}