From 5997487fc5cc2126fb5afa12b75b67f3985e11d2 Mon Sep 17 00:00:00 2001 From: rtz12 Date: Sun, 2 Oct 2016 04:59:08 +0200 Subject: Importer etwas netter strukturiert und Support fuer die neuen ACD-Daten hinzugefuegt diff --git a/.gitignore b/.gitignore index cb1aa4a..76e7090 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,7 @@ package.json #Gebuildete Dateien assets/js/* -assets/css/* \ No newline at end of file +assets/css/* + +#Binaries +/importer diff --git a/Makefile b/Makefile index 9fbe1b7..fbef6fa 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,7 @@ include config.mk +IMPORTER_FILES = $(shell find tools/importer/ -type f -name '*.go') + CSS_FILES = $(patsubst assets_src/%, assets/%, $(shell find assets_src/css/ -type f -name '*.css')) LESS_FILES = $(patsubst assets_src/%.less, assets/%.css, $(shell find assets_src/css/ -type f -name '*.less')) JS_FILES = $(patsubst assets_src/%, assets/%, $(shell find assets_src/js/ -maxdepth 1 -type f -name '*.js')) @@ -48,6 +50,7 @@ clean: -rm -- $(LESS_FILES) -rm -- $(JS_FILES) -rm -- $(DEPFILES) + -rm -- importer superclean: clean -rm -- .npm_update @@ -59,5 +62,8 @@ ultraclean: superclean run: all go run *.go +importer: $(IMPORTER_FILES) + go build -o importer tools/importer/*.go + .PHONY: all print_info clean superclean ultraclean run diff --git a/tools/importer/.gitignore b/tools/importer/.gitignore new file mode 100644 index 0000000..a6c57f5 --- /dev/null +++ b/tools/importer/.gitignore @@ -0,0 +1 @@ +*.json diff --git a/tools/importer/ACDv2Reader.go b/tools/importer/ACDv2Reader.go new file mode 100644 index 0000000..2ddca4d --- /dev/null +++ b/tools/importer/ACDv2Reader.go @@ -0,0 +1,66 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "path/filepath" + "strings" + + "fagott.pw/grilist/modules/grils" +) + +type ACDv2Reader struct{} + +type ACDv2NameValue struct { + Name string `json:"name"` + Value string `json:"value"` +} + +type ACDv2Gril struct { + Name struct { + Romaji string `json:"romaji"` + Japanese string `json:"japanese"` + Aliases []string `json:"aliases"` + } `json:"name"` + Image struct { + Thumbnail string `json:"thumb"` + Full string `json:"full"` + } `json:"image"` + Tags []struct { + ID int `json:"id"` + Name string `json:"name"` + } `json:"tags"` + Traits struct { + Official []ACDv2NameValue `json:"official"` + Indexed []ACDv2NameValue `json:"indexed"` + } `json:"traits"` + Assignments []int `json:"assignments"` + CharsWithSimilarTraits []int `json:"chars_similar_traits"` + Extra []ACDv2NameValue `json:"extra"` + Role string `json:"role"` +} + +func (r *ACDv2Reader) Read(path string) WrappedGril { + g := WrappedGril{} + g.Gril = grils.Gril{} + data, err := ioutil.ReadFile(path) + ifErrExit(err) + var jObj ACDv2Gril + err = json.Unmarshal(data, &jObj) + ifErrExit(err) + g.Image = jObj.Image.Full + g.Thumb = jObj.Image.Thumbnail + g.Gril.ID = toInt(strings.TrimSuffix(filepath.Base(path), ".json")) + g.Gril.KanjiName = jObj.Name.Japanese + g.Gril.RomajiName = jObj.Name.Romaji + g.Gril.OtherNames = jObj.Name.Aliases + g.Gril.Tags = make([]string, 0) + for _, v := range jObj.Tags { + g.Gril.Tags = append(g.Gril.Tags, v.Name) + } + return g +} + +func (r *ACDv2Reader) ID() int { + return int(grils.DataSourceACD) +} diff --git a/tools/importer/main.go b/tools/importer/main.go index 9caf287..ba98fb5 100644 --- a/tools/importer/main.go +++ b/tools/importer/main.go @@ -13,6 +13,13 @@ import ( _ "github.com/lib/pq" ) +var path string +var sourceType string +var imgUrlExportFile string +var exportFile *os.File +var db *sql.DB +var r GrilReader + type WrappedGril struct { Gril grils.Gril Image string @@ -28,6 +35,8 @@ func GetGrilReader(rType string) GrilReader { switch rType { case "ACD": return &ACDReader{} + case "ACDv2": + return &ACDv2Reader{} } return nil } @@ -38,15 +47,100 @@ func LogErr(err error) { } } +func TryGetExistingGril(g WrappedGril, idmap map[int]int) int { + dbID, _ := idmap[g.Gril.ID] + return dbID +} + +func InsertGril(g WrappedGril) int { + var dbID int + row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;") + row.Scan(&dbID) + fmt.Printf( + "Assigned %d from %s to %d\n", + g.Gril.ID, + sourceType, + dbID) + db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id, + source, source_id) VALUES ($1, $2, $3);`, + dbID, r.ID(), g.Gril.ID) + if g.Gril.KanjiName != "" { + _, err := db.Exec(`INSERT INTO + grilist.gril_names (gril_id, name, name_type) + VALUES ($1, $2, $3);`, dbID, g.Gril.KanjiName, + 0) + LogErr(err) + } + if g.Gril.RomajiName != "" { + _, err := db.Exec(`INSERT INTO + grilist.gril_names (gril_id, name, name_type) + VALUES ($1, $2, $3);`, dbID, g.Gril.RomajiName, + 1) + LogErr(err) + } + for _, v := range g.Gril.OtherNames { + _, err := db.Exec(`INSERT INTO + grilist.gril_names (gril_id, name, name_type) + VALUES ($1, $2, $3);`, dbID, v, 2) + LogErr(err) + } + if exportFile != nil { + if g.Image != "" { + fmt.Fprintf( + exportFile, + "%d %d %s\n", + dbID, + 0, + g.Image) + } + if g.Thumb != "" { + fmt.Fprintf( + exportFile, + "%d %d %s\n", + dbID, + 1, + g.Thumb) + } + } + fmt.Printf("Inserted %s\n", g.Gril.RomajiName) + return dbID +} + +func InsertTags(dbID int, g WrappedGril, taglist map[string]int) { + for _, v := range g.Gril.Tags { + if _, ok := taglist[v]; ok { + continue + } + var id int + row := db.QueryRow("INSERT INTO grilist.tags (name) VALUES ($1) RETURNING id;", v) + row.Scan(&id) + taglist[v] = id + fmt.Printf("Inserted tag %s as %d\n", v, id) + } + rows, err := db.Query("SELECT tag_id FROM grils_tags WHERE gril_id = $1;", dbID) + LogErr(err) + existingTags := make(map[int]bool, 0) + for rows.Next() { + var tagID int + rows.Scan(&tagID) + existingTags[tagID] = true + } + for _, v := range g.Gril.Tags { + tagID := taglist[v] + if _, ok := existingTags[tagID]; ok { + continue + } + _, err := db.Exec(`INSERT INTO grilist.grils_tags (gril_id, tag_id) VALUES ($1, $2);`, dbID, tagID) + fmt.Printf("Assigned tag %s to %s\n", v, g.Gril.RomajiName) + LogErr(err) + } +} + func main() { - var path string - var sourceType string - var imgUrlExportFile string flag.StringVar(&path, "path", "", "path of the source files") flag.StringVar(&sourceType, "type", "", "type of the files (ACD, AniDB)") flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!") flag.Parse() - var exportFile *os.File if imgUrlExportFile != "" { var err error if exportFile, err = os.Create(imgUrlExportFile); err != nil { @@ -60,14 +154,15 @@ func main() { flag.Usage() os.Exit(1) } - r := GetGrilReader(sourceType) + r = GetGrilReader(sourceType) if r == nil { fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid gril source!") flag.Usage() os.Exit(1) } config := grilist.LoadConfig() - db, err := sql.Open("postgres", config.DBConnectionString()) + var err error + db, err = sql.Open("postgres", config.DBConnectionString()) if err == nil { err = db.Ping() } @@ -76,6 +171,7 @@ func main() { fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } + taglist := make(map[string]int) rows, _ := db.Query("SELECT id, name FROM grilist.tags;") for rows.Next() { @@ -92,82 +188,19 @@ func main() { rows.Scan(&sourceID, &grilID) idmap[sourceID] = grilID } + filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if info.IsDir() { return nil } g := r.Read(path) + var dbID int - var ok bool - if dbID, ok = idmap[g.Gril.ID]; !ok { - row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;") - row.Scan(&dbID) - fmt.Printf( - "Assigned %d from %s to %d\n", - g.Gril.ID, - sourceType, - dbID) - db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id, - source, source_id) VALUES ($1, $2, $3);`, - dbID, r.ID(), g.Gril.ID) - if g.Gril.KanjiName != "" { - _, err := db.Exec(`INSERT INTO - grilist.gril_names (gril_id, name, name_type) - VALUES ($1, $2, $3);`, dbID, g.Gril.KanjiName, - 0) - LogErr(err) - } - if g.Gril.RomajiName != "" { - _, err := db.Exec(`INSERT INTO - grilist.gril_names (gril_id, name, name_type) - VALUES ($1, $2, $3);`, dbID, g.Gril.RomajiName, - 1) - LogErr(err) - } - for _, v := range g.Gril.OtherNames { - _, err := db.Exec(`INSERT INTO - grilist.gril_names (gril_id, name, name_type) - VALUES ($1, $2, $3);`, dbID, v, 2) - LogErr(err) - } - if exportFile != nil { - if g.Image != "" { - fmt.Fprintf( - exportFile, - "%d %d %s\n", - dbID, - 0, - g.Image) - } - if g.Thumb != "" { - fmt.Fprintf( - exportFile, - "%d %d %s\n", - dbID, - 1, - g.Thumb) - } - } - fmt.Printf("Inserted %s\n", g.Gril.RomajiName) - } - for _, v := range g.Gril.Tags { - if _, ok := taglist[v]; ok { - continue - } - var id int - row := db.QueryRow("INSERT INTO grilist.tags (name) VALUES ($1) RETURNING id;", v) - row.Scan(&id) - taglist[v] = id - } - _, err = db.Exec(`DELETE FROM grilist.grils_tags WHERE gril_id - = $1`, dbID) - LogErr(err) - for _, v := range g.Gril.Tags { - tagID := taglist[v] - _, err := db.Exec(`INSERT INTO grilist.grils_tags - (gril_id, tag_id) VALUES ($1, $2);`, dbID, tagID) - LogErr(err) + if dbID = TryGetExistingGril(g, idmap); dbID == 0 { + dbID = InsertGril(g) } + InsertTags(dbID, g, taglist) + return nil }) } -- cgit v0.10.1