aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorrtz12 <koenig@fagott.pw>2015-12-28 18:56:16 (UTC)
committerrtz12 <koenig@fagott.pw>2015-12-28 18:56:16 (UTC)
commit23f73a60a4cf4621dd5951bfe6601a8c154cad07 (patch)
tree91604c8fb001c93f0a3e021624d1b2ad22d7e24f
parent95d733307cc87610101469953e61435dbcbeb052 (diff)
Grilimporter hinzugefuegt
Diffstat
-rw-r--r--tools/importer/ACDReader.go81
-rw-r--r--tools/importer/main.go152
2 files changed, 233 insertions, 0 deletions
diff --git a/tools/importer/ACDReader.go b/tools/importer/ACDReader.go
new file mode 100644
index 0000000..694087e
--- /dev/null
+++ b/tools/importer/ACDReader.go
@@ -0,0 +1,81 @@
1package main
2
3import (
4 "encoding/json"
5 "fmt"
6 "io/ioutil"
7 "os"
8 "regexp"
9 "strconv"
10 "strings"
11
12 "fagott.pw/grilist/modules/grils"
13)
14
15var tagRegex = regexp.MustCompile(`\<a.*?\>(.+)\<\/a\>`)
16
17type ACDReader struct{}
18
19func ifErrExit(err error) {
20 if err != nil {
21 fmt.Fprintf(os.Stderr, "%v\n", err)
22 os.Exit(1)
23 }
24}
25
26func toInt(s string) int {
27 i, err := strconv.ParseInt(s, 10, 32)
28 ifErrExit(err)
29 return int(i)
30}
31
32func (r *ACDReader) Read(path string) WrappedGril {
33 g := WrappedGril{}
34 g.Gril = grils.Gril{}
35 data, err := ioutil.ReadFile(path)
36 ifErrExit(err)
37 var jObj interface{}
38 err = json.Unmarshal(data, &jObj)
39 ifErrExit(err)
40 for k, v := range jObj.(map[string]interface{}) {
41 switch k {
42 case "Character ID":
43 g.Gril.ID = toInt(v.(string))
44 case "Romaji Name":
45 g.Gril.RomajiName = v.(string)
46 case "Japanese Name":
47 g.Gril.KanjiName = v.(string)
48 case "Aliases":
49 g.Gril.OtherNames = make([]string, 0)
50 for _, n := range strings.Split(v.(string), ",") {
51 if n == "" {
52 continue
53 }
54 g.Gril.OtherNames = append(
55 g.Gril.OtherNames,
56 strings.TrimSpace(n))
57 }
58 case "Tagged":
59 g.Gril.Tags = make([]string, 0)
60 for _, n := range strings.Split(v.(string), ",") {
61 if n == "" {
62 continue
63 }
64 tag := strings.TrimSpace(n)
65 tag = tagRegex.FindStringSubmatch(tag)[1]
66 g.Gril.Tags = append(
67 g.Gril.Tags,
68 tag)
69 }
70 case "__img":
71 g.Image = v.(string)
72 case "__thumb":
73 g.Thumb = v.(string)
74 }
75 }
76 return g
77}
78
79func (r *ACDReader) ID() int {
80 return int(grils.DataSourceACD)
81}
diff --git a/tools/importer/main.go b/tools/importer/main.go
new file mode 100644
index 0000000..41733a1
--- /dev/null
+++ b/tools/importer/main.go
@@ -0,0 +1,152 @@
1package main
2
3import (
4 "database/sql"
5 "flag"
6 "fmt"
7 "os"
8 "path/filepath"
9
10 "fagott.pw/grilist/grilist"
11 "fagott.pw/grilist/modules/grils"
12
13 _ "github.com/lib/pq"
14)
15
16type WrappedGril struct {
17 Gril grils.Gril
18 Image string
19 Thumb string
20}
21
22type GrilReader interface {
23 Read(path string) WrappedGril
24 ID() int
25}
26
27func GetGrilReader(rType string) GrilReader {
28 switch rType {
29 case "ACD":
30 return &ACDReader{}
31 }
32 return nil
33}
34
35func main() {
36 var path string
37 var sourceType string
38 var imgUrlExportFile string
39 flag.StringVar(&path, "path", "", "path of the source files")
40 flag.StringVar(&sourceType, "type", "", "type of the files (ACD, AniDB)")
41 flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!")
42 flag.Parse()
43 var exportFile *os.File
44 if imgUrlExportFile != "" {
45 var err error
46 if exportFile, err = os.Create(imgUrlExportFile); err != nil {
47 fmt.Fprintf(os.Stderr, "%s\n", "error while accessing \"%s\"")
48 fmt.Fprintf(os.Stderr, "%v\n", err)
49 os.Exit(1)
50 }
51 }
52 if path == "" {
53 fmt.Fprintf(os.Stderr, "%s\n", "\"path\" must be specified!")
54 flag.Usage()
55 os.Exit(1)
56 }
57 r := GetGrilReader(sourceType)
58 if r == nil {
59 fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid gril source!")
60 flag.Usage()
61 os.Exit(1)
62 }
63 config := grilist.LoadConfig()
64 db, err := sql.Open("postgres", config.DBConnectionString())
65 if err == nil {
66 err = db.Ping()
67 }
68 if err != nil {
69 fmt.Fprintf(os.Stderr, "%s\n", "error with database connection")
70 fmt.Fprintf(os.Stderr, "%v\n", err)
71 os.Exit(1)
72 }
73 taglist := make(map[string]int)
74 rows, _ := db.Query("SELECT id, name FROM grilist.tags;")
75 for rows.Next() {
76 var id int
77 var name string
78 rows.Scan(&id)
79 rows.Scan(&name)
80 taglist[name] = id
81 }
82 idmap := make(map[int]int)
83 rows, _ = db.Query("SELECT source_id, gril_id FROM grilist.grils_id_mappings WHERE source = $1;", r.ID())
84 for rows.Next() {
85 var sourceID int
86 var grilID int
87 rows.Scan(&sourceID)
88 rows.Scan(&grilID)
89 idmap[sourceID] = grilID
90 }
91 filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
92 if info.IsDir() {
93 return nil
94 }
95 g := r.Read(path)
96 for _, v := range g.Gril.Tags {
97 if _, ok := taglist[v]; ok {
98 continue
99 }
100 var id int
101 row := db.QueryRow("INSERT INTO grilist.tags (name) VALUES ($1) RETURNING id;", v)
102 row.Scan(&id)
103 taglist[v] = id
104 }
105 if _, ok := idmap[g.Gril.ID]; !ok {
106 row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;")
107 var dbID int
108 row.Scan(&dbID)
109 for _, v := range g.Gril.Tags {
110 tagID := taglist[v]
111 db.Exec("INSERT INTO grilist.grils_tags (gril_id, tag_id) VALUES ($1, $2);", dbID, tagID)
112 }
113 db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id,
114 source, source_id) VALUES ($1, $2, $3);`,
115 dbID, r.ID(), g.Gril.ID)
116 if g.Gril.KanjiName != "" {
117 db.Exec(`INSERT INTO grilist.gril_names
118 (gril_id, name, name_type) VALUES ($1, $2,
119 $3);`, dbID, g.Gril.KanjiName, 0)
120 }
121 if g.Gril.RomajiName != "" {
122 db.Exec(`INSERT INTO grilist.gril_names
123 (gril_id, name, name_type) VALUES ($1, $2,
124 $3);`, dbID, g.Gril.RomajiName, 1)
125 }
126 for _, v := range g.Gril.OtherNames {
127 db.Exec(`INSERT INTO grilist.gril_names
128 (gril_id, name, name_type) VALUES ($1, $2,
129 $3);`, dbID, v, 2)
130 }
131 if exportFile != nil {
132 if g.Image != "" {
133 fmt.Fprintf(
134 exportFile,
135 "%d %d %s\n",
136 dbID,
137 0,
138 g.Image)
139 }
140 if g.Thumb != "" {
141 fmt.Fprintf(
142 exportFile,
143 "%d %d %s\n",
144 dbID,
145 1,
146 g.Thumb)
147 }
148 }
149 }
150 return nil
151 })
152}
generated by cgit v0.10.1 at 2024-12-26 20:13:18 (UTC)