aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorrtz12 <koenig@fagott.pw>2016-10-02 02:59:08 (UTC)
committerrtz12 <koenig@fagott.pw>2016-10-02 02:59:08 (UTC)
commit5997487fc5cc2126fb5afa12b75b67f3985e11d2 (patch)
treed0631885961a35daf7ec302161ea888ae02183aa
parentb799217c66df7c1c4ef285699273a5222efccfb0 (diff)
Importer etwas netter strukturiert und Support fuer die neuen ACD-Daten
hinzugefuegt
Diffstat
-rw-r--r--.gitignore5
-rw-r--r--Makefile6
-rw-r--r--tools/importer/.gitignore1
-rw-r--r--tools/importer/ACDv2Reader.go66
-rw-r--r--tools/importer/main.go183
5 files changed, 185 insertions, 76 deletions
diff --git a/.gitignore b/.gitignore
index cb1aa4a..76e7090 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,4 +8,7 @@ package.json
8 8
9#Gebuildete Dateien 9#Gebuildete Dateien
10assets/js/* 10assets/js/*
11assets/css/* \ No newline at end of file 11assets/css/*
12
13#Binaries
14/importer
diff --git a/Makefile b/Makefile
index 9fbe1b7..fbef6fa 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,7 @@
1include config.mk 1include config.mk
2 2
3IMPORTER_FILES = $(shell find tools/importer/ -type f -name '*.go')
4
3CSS_FILES = $(patsubst assets_src/%, assets/%, $(shell find assets_src/css/ -type f -name '*.css')) 5CSS_FILES = $(patsubst assets_src/%, assets/%, $(shell find assets_src/css/ -type f -name '*.css'))
4LESS_FILES = $(patsubst assets_src/%.less, assets/%.css, $(shell find assets_src/css/ -type f -name '*.less')) 6LESS_FILES = $(patsubst assets_src/%.less, assets/%.css, $(shell find assets_src/css/ -type f -name '*.less'))
5JS_FILES = $(patsubst assets_src/%, assets/%, $(shell find assets_src/js/ -maxdepth 1 -type f -name '*.js')) 7JS_FILES = $(patsubst assets_src/%, assets/%, $(shell find assets_src/js/ -maxdepth 1 -type f -name '*.js'))
@@ -48,6 +50,7 @@ clean:
48 -rm -- $(LESS_FILES) 50 -rm -- $(LESS_FILES)
49 -rm -- $(JS_FILES) 51 -rm -- $(JS_FILES)
50 -rm -- $(DEPFILES) 52 -rm -- $(DEPFILES)
53 -rm -- importer
51 54
52superclean: clean 55superclean: clean
53 -rm -- .npm_update 56 -rm -- .npm_update
@@ -59,5 +62,8 @@ ultraclean: superclean
59run: all 62run: all
60 go run *.go 63 go run *.go
61 64
65importer: $(IMPORTER_FILES)
66 go build -o importer tools/importer/*.go
67
62.PHONY: all print_info clean superclean ultraclean run 68.PHONY: all print_info clean superclean ultraclean run
63 69
diff --git a/tools/importer/.gitignore b/tools/importer/.gitignore
new file mode 100644
index 0000000..a6c57f5
--- /dev/null
+++ b/tools/importer/.gitignore
@@ -0,0 +1 @@
*.json
diff --git a/tools/importer/ACDv2Reader.go b/tools/importer/ACDv2Reader.go
new file mode 100644
index 0000000..2ddca4d
--- /dev/null
+++ b/tools/importer/ACDv2Reader.go
@@ -0,0 +1,66 @@
1package main
2
3import (
4 "encoding/json"
5 "io/ioutil"
6 "path/filepath"
7 "strings"
8
9 "fagott.pw/grilist/modules/grils"
10)
11
12type ACDv2Reader struct{}
13
14type ACDv2NameValue struct {
15 Name string `json:"name"`
16 Value string `json:"value"`
17}
18
19type ACDv2Gril struct {
20 Name struct {
21 Romaji string `json:"romaji"`
22 Japanese string `json:"japanese"`
23 Aliases []string `json:"aliases"`
24 } `json:"name"`
25 Image struct {
26 Thumbnail string `json:"thumb"`
27 Full string `json:"full"`
28 } `json:"image"`
29 Tags []struct {
30 ID int `json:"id"`
31 Name string `json:"name"`
32 } `json:"tags"`
33 Traits struct {
34 Official []ACDv2NameValue `json:"official"`
35 Indexed []ACDv2NameValue `json:"indexed"`
36 } `json:"traits"`
37 Assignments []int `json:"assignments"`
38 CharsWithSimilarTraits []int `json:"chars_similar_traits"`
39 Extra []ACDv2NameValue `json:"extra"`
40 Role string `json:"role"`
41}
42
43func (r *ACDv2Reader) Read(path string) WrappedGril {
44 g := WrappedGril{}
45 g.Gril = grils.Gril{}
46 data, err := ioutil.ReadFile(path)
47 ifErrExit(err)
48 var jObj ACDv2Gril
49 err = json.Unmarshal(data, &jObj)
50 ifErrExit(err)
51 g.Image = jObj.Image.Full
52 g.Thumb = jObj.Image.Thumbnail
53 g.Gril.ID = toInt(strings.TrimSuffix(filepath.Base(path), ".json"))
54 g.Gril.KanjiName = jObj.Name.Japanese
55 g.Gril.RomajiName = jObj.Name.Romaji
56 g.Gril.OtherNames = jObj.Name.Aliases
57 g.Gril.Tags = make([]string, 0)
58 for _, v := range jObj.Tags {
59 g.Gril.Tags = append(g.Gril.Tags, v.Name)
60 }
61 return g
62}
63
64func (r *ACDv2Reader) ID() int {
65 return int(grils.DataSourceACD)
66}
diff --git a/tools/importer/main.go b/tools/importer/main.go
index 9caf287..ba98fb5 100644
--- a/tools/importer/main.go
+++ b/tools/importer/main.go
@@ -13,6 +13,13 @@ import (
13 _ "github.com/lib/pq" 13 _ "github.com/lib/pq"
14) 14)
15 15
16var path string
17var sourceType string
18var imgUrlExportFile string
19var exportFile *os.File
20var db *sql.DB
21var r GrilReader
22
16type WrappedGril struct { 23type WrappedGril struct {
17 Gril grils.Gril 24 Gril grils.Gril
18 Image string 25 Image string
@@ -28,6 +35,8 @@ func GetGrilReader(rType string) GrilReader {
28 switch rType { 35 switch rType {
29 case "ACD": 36 case "ACD":
30 return &ACDReader{} 37 return &ACDReader{}
38 case "ACDv2":
39 return &ACDv2Reader{}
31 } 40 }
32 return nil 41 return nil
33} 42}
@@ -38,15 +47,100 @@ func LogErr(err error) {
38 } 47 }
39} 48}
40 49
50func TryGetExistingGril(g WrappedGril, idmap map[int]int) int {
51 dbID, _ := idmap[g.Gril.ID]
52 return dbID
53}
54
55func InsertGril(g WrappedGril) int {
56 var dbID int
57 row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;")
58 row.Scan(&dbID)
59 fmt.Printf(
60 "Assigned %d from %s to %d\n",
61 g.Gril.ID,
62 sourceType,
63 dbID)
64 db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id,
65 source, source_id) VALUES ($1, $2, $3);`,
66 dbID, r.ID(), g.Gril.ID)
67 if g.Gril.KanjiName != "" {
68 _, err := db.Exec(`INSERT INTO
69 grilist.gril_names (gril_id, name, name_type)
70 VALUES ($1, $2, $3);`, dbID, g.Gril.KanjiName,
71 0)
72 LogErr(err)
73 }
74 if g.Gril.RomajiName != "" {
75 _, err := db.Exec(`INSERT INTO
76 grilist.gril_names (gril_id, name, name_type)
77 VALUES ($1, $2, $3);`, dbID, g.Gril.RomajiName,
78 1)
79 LogErr(err)
80 }
81 for _, v := range g.Gril.OtherNames {
82 _, err := db.Exec(`INSERT INTO
83 grilist.gril_names (gril_id, name, name_type)
84 VALUES ($1, $2, $3);`, dbID, v, 2)
85 LogErr(err)
86 }
87 if exportFile != nil {
88 if g.Image != "" {
89 fmt.Fprintf(
90 exportFile,
91 "%d %d %s\n",
92 dbID,
93 0,
94 g.Image)
95 }
96 if g.Thumb != "" {
97 fmt.Fprintf(
98 exportFile,
99 "%d %d %s\n",
100 dbID,
101 1,
102 g.Thumb)
103 }
104 }
105 fmt.Printf("Inserted %s\n", g.Gril.RomajiName)
106 return dbID
107}
108
109func InsertTags(dbID int, g WrappedGril, taglist map[string]int) {
110 for _, v := range g.Gril.Tags {
111 if _, ok := taglist[v]; ok {
112 continue
113 }
114 var id int
115 row := db.QueryRow("INSERT INTO grilist.tags (name) VALUES ($1) RETURNING id;", v)
116 row.Scan(&id)
117 taglist[v] = id
118 fmt.Printf("Inserted tag %s as %d\n", v, id)
119 }
120 rows, err := db.Query("SELECT tag_id FROM grils_tags WHERE gril_id = $1;", dbID)
121 LogErr(err)
122 existingTags := make(map[int]bool, 0)
123 for rows.Next() {
124 var tagID int
125 rows.Scan(&tagID)
126 existingTags[tagID] = true
127 }
128 for _, v := range g.Gril.Tags {
129 tagID := taglist[v]
130 if _, ok := existingTags[tagID]; ok {
131 continue
132 }
133 _, err := db.Exec(`INSERT INTO grilist.grils_tags (gril_id, tag_id) VALUES ($1, $2);`, dbID, tagID)
134 fmt.Printf("Assigned tag %s to %s\n", v, g.Gril.RomajiName)
135 LogErr(err)
136 }
137}
138
41func main() { 139func main() {
42 var path string
43 var sourceType string
44 var imgUrlExportFile string
45 flag.StringVar(&path, "path", "", "path of the source files") 140 flag.StringVar(&path, "path", "", "path of the source files")
46 flag.StringVar(&sourceType, "type", "", "type of the files (ACD, AniDB)") 141 flag.StringVar(&sourceType, "type", "", "type of the files (ACD, AniDB)")
47 flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!") 142 flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!")
48 flag.Parse() 143 flag.Parse()
49 var exportFile *os.File
50 if imgUrlExportFile != "" { 144 if imgUrlExportFile != "" {
51 var err error 145 var err error
52 if exportFile, err = os.Create(imgUrlExportFile); err != nil { 146 if exportFile, err = os.Create(imgUrlExportFile); err != nil {
@@ -60,14 +154,15 @@ func main() {
60 flag.Usage() 154 flag.Usage()
61 os.Exit(1) 155 os.Exit(1)
62 } 156 }
63 r := GetGrilReader(sourceType) 157 r = GetGrilReader(sourceType)
64 if r == nil { 158 if r == nil {
65 fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid gril source!") 159 fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid gril source!")
66 flag.Usage() 160 flag.Usage()
67 os.Exit(1) 161 os.Exit(1)
68 } 162 }
69 config := grilist.LoadConfig() 163 config := grilist.LoadConfig()
70 db, err := sql.Open("postgres", config.DBConnectionString()) 164 var err error
165 db, err = sql.Open("postgres", config.DBConnectionString())
71 if err == nil { 166 if err == nil {
72 err = db.Ping() 167 err = db.Ping()
73 } 168 }
@@ -76,6 +171,7 @@ func main() {
76 fmt.Fprintf(os.Stderr, "%v\n", err) 171 fmt.Fprintf(os.Stderr, "%v\n", err)
77 os.Exit(1) 172 os.Exit(1)
78 } 173 }
174
79 taglist := make(map[string]int) 175 taglist := make(map[string]int)
80 rows, _ := db.Query("SELECT id, name FROM grilist.tags;") 176 rows, _ := db.Query("SELECT id, name FROM grilist.tags;")
81 for rows.Next() { 177 for rows.Next() {
@@ -92,82 +188,19 @@ func main() {
92 rows.Scan(&sourceID, &grilID) 188 rows.Scan(&sourceID, &grilID)
93 idmap[sourceID] = grilID 189 idmap[sourceID] = grilID
94 } 190 }
191
95 filepath.Walk(path, func(path string, info os.FileInfo, err error) error { 192 filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
96 if info.IsDir() { 193 if info.IsDir() {
97 return nil 194 return nil
98 } 195 }
99 g := r.Read(path) 196 g := r.Read(path)
197
100 var dbID int 198 var dbID int
101 var ok bool 199 if dbID = TryGetExistingGril(g, idmap); dbID == 0 {
102 if dbID, ok = idmap[g.Gril.ID]; !ok { 200 dbID = InsertGril(g)
103 row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;")
104 row.Scan(&dbID)
105 fmt.Printf(
106 "Assigned %d from %s to %d\n",
107 g.Gril.ID,
108 sourceType,
109 dbID)
110 db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id,
111 source, source_id) VALUES ($1, $2, $3);`,
112 dbID, r.ID(), g.Gril.ID)
113 if g.Gril.KanjiName != "" {
114 _, err := db.Exec(`INSERT INTO
115 grilist.gril_names (gril_id, name, name_type)
116 VALUES ($1, $2, $3);`, dbID, g.Gril.KanjiName,
117 0)
118 LogErr(err)
119 }
120 if g.Gril.RomajiName != "" {
121 _, err := db.Exec(`INSERT INTO
122 grilist.gril_names (gril_id, name, name_type)
123 VALUES ($1, $2, $3);`, dbID, g.Gril.RomajiName,
124 1)
125 LogErr(err)
126 }
127 for _, v := range g.Gril.OtherNames {
128 _, err := db.Exec(`INSERT INTO
129 grilist.gril_names (gril_id, name, name_type)
130 VALUES ($1, $2, $3);`, dbID, v, 2)
131 LogErr(err)
132 }
133 if exportFile != nil {
134 if g.Image != "" {
135 fmt.Fprintf(
136 exportFile,
137 "%d %d %s\n",
138 dbID,
139 0,
140 g.Image)
141 }
142 if g.Thumb != "" {
143 fmt.Fprintf(
144 exportFile,
145 "%d %d %s\n",
146 dbID,
147 1,
148 g.Thumb)
149 }
150 }
151 fmt.Printf("Inserted %s\n", g.Gril.RomajiName)
152 }
153 for _, v := range g.Gril.Tags {
154 if _, ok := taglist[v]; ok {
155 continue
156 }
157 var id int
158 row := db.QueryRow("INSERT INTO grilist.tags (name) VALUES ($1) RETURNING id;", v)
159 row.Scan(&id)
160 taglist[v] = id
161 }
162 _, err = db.Exec(`DELETE FROM grilist.grils_tags WHERE gril_id
163 = $1`, dbID)
164 LogErr(err)
165 for _, v := range g.Gril.Tags {
166 tagID := taglist[v]
167 _, err := db.Exec(`INSERT INTO grilist.grils_tags
168 (gril_id, tag_id) VALUES ($1, $2);`, dbID, tagID)
169 LogErr(err)
170 } 201 }
202 InsertTags(dbID, g, taglist)
203
171 return nil 204 return nil
172 }) 205 })
173} 206}
generated by cgit v0.10.1 at 2024-12-25 17:58:30 (UTC)