1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
|
package crawler
import (
"log"
"strconv"
"strings"
"github.com/PuerkitoBio/goquery"
)
type ACDCrawler struct{}
func (a ACDCrawler) Name() string {
return "ACD"
}
func (a ACDCrawler) Crawl(id int) (CharacterData, error) {
c := make(CharacterData)
doc, err := goquery.NewDocument("http://www.animecharactersdatabase.com/character.php?id=" + strconv.Itoa(id))
if err != nil {
log.Println(err)
return nil, CrawlError
}
text := doc.Text()
if strings.Contains(text, "bad character : try") {
return nil, CharacterNotFound
}
if strings.Contains(text, "Your IP has been blocked") {
return nil, Banned
}
dataTr := doc.Find("#besttable").Next().ChildrenFiltered("tbody").ChildrenFiltered("tr")
leftSide := dataTr.Children().Eq(0)
rightSide := dataTr.Children().Eq(1)
imageCols := rightSide.ChildrenFiltered("table").First().Find("tbody > tr > td")
val, _ := imageCols.Eq(0).Find("img").Attr("src")
c["__thumb"] = val
val, _ = imageCols.Eq(1).Find("a").Attr("href")
c["__img"] = val
leftSide.ChildrenFiltered("table").Eq(1).Find("tr").Each(func(i int, s *goquery.Selection) {
c[s.Find("th").Text()] = s.Find("td").Text()
})
var key string
leftSide.
ChildrenFiltered("table").
Eq(0).
Find("td").
Eq(1).
Find("dl").
Children().
Each(func(i int, s *goquery.Selection) {
switch goquery.NodeName(s) {
case "dt":
key = s.Text()
case "dd":
c[key] = s.Text()
}
})
tags := make([]string, 0)
leftSide.ChildrenFiltered("div").Eq(0).Find("a").Each(func(i int, s *goquery.Selection) {
tags = append(tags, s.Text())
})
c["__tags"] = tags
vas := make([]string, 0)
leftSide.ChildrenFiltered("div").Eq(1).Find("a").Each(func(i int, s *goquery.Selection) {
vas = append(vas, s.Text())
})
c["__vas"] = vas
leftSide.ChildrenFiltered("dl").Children().Each(func(i int, s *goquery.Selection) {
switch goquery.NodeName(s) {
case "dt":
key = s.Text()
case "dd":
c[key] = s.Text()
}
})
apps := make([]int, 0)
rightSide.Find(".tile3top").Each(func(i int, s *goquery.Selection) {
val, _ = s.Find("a").Attr("href")
id, _ := strconv.Atoi(strings.Split(val, "=")[1])
apps = append(apps, id)
})
c["__appearances"] = apps
return c, nil
}
|
