From 4a383de2f73d67bb48f5a66c7a1e8cbf46e2fd88 Mon Sep 17 00:00:00 2001 From: rtz12 Date: Sun, 9 Oct 2016 16:41:57 +0200 Subject: Animeimporter hinzugefuegt diff --git a/modules/series/series.go b/modules/series/series.go new file mode 100644 index 0000000..1f50903 --- /dev/null +++ b/modules/series/series.go @@ -0,0 +1,11 @@ +package series + +type Series struct { + ID int + Name string + KanjiName string + RomajiName string + FuriganaName string + OtherNames []string + Tags []string +} diff --git a/modules/tags/tag.go b/modules/tags/tag.go new file mode 100644 index 0000000..026df0f --- /dev/null +++ b/modules/tags/tag.go @@ -0,0 +1,8 @@ +package tags + +type TagType int + +const ( + TagTypeGril TagType = iota + TagTypeSeries +) diff --git a/tools/animeimporter/ACDv2Reader.go b/tools/animeimporter/ACDv2Reader.go new file mode 100644 index 0000000..36564e4 --- /dev/null +++ b/tools/animeimporter/ACDv2Reader.go @@ -0,0 +1,59 @@ +package main + +import ( + "encoding/json" + "io/ioutil" + "path/filepath" + "strconv" + "strings" + + "fagott.pw/grilist/modules/grils" +) + +func ifErrExit(err error) { + if err != nil { + panic(err) + } +} + +func toInt(s string) int { + i, err := strconv.ParseInt(s, 10, 32) + ifErrExit(err) + return int(i) +} + +type ACDv2Reader struct{} + +type ACDv2Series struct { + Name struct { + English string `json:"english"` + Romaji string `json:"romaji"` + Japanese string `json:"japanese"` + Furigana string `json:"furigana"` + Aliases []string `json:"aliases"` + } `json:"name"` + Tags []string `json:"tags"` + Image string `json:"image"` +} + +func (r *ACDv2Reader) Read(path string) WrappedSeries { + s := WrappedSeries{} + data, err := ioutil.ReadFile(path) + ifErrExit(err) + var jObj ACDv2Series + err = json.Unmarshal(data, &jObj) + ifErrExit(err) + s.Image = jObj.Image + s.Series.ID = toInt(strings.TrimSuffix(filepath.Base(path), ".json")) + s.Series.Name = jObj.Name.English + s.Series.KanjiName = jObj.Name.Japanese + s.Series.RomajiName = jObj.Name.Romaji + s.Series.FuriganaName = jObj.Name.Furigana + s.Series.OtherNames = jObj.Name.Aliases + s.Series.Tags = jObj.Tags + return s +} + +func (r *ACDv2Reader) ID() int { + return int(grils.DataSourceACD) +} diff --git a/tools/animeimporter/main.go b/tools/animeimporter/main.go new file mode 100644 index 0000000..7d95b8e --- /dev/null +++ b/tools/animeimporter/main.go @@ -0,0 +1,202 @@ +package main + +import ( + "database/sql" + "flag" + "fmt" + "os" + "path/filepath" + + "fagott.pw/grilist/grilist" + "fagott.pw/grilist/modules/series" + "fagott.pw/grilist/modules/tags" +) + +var path string +var sourceType string +var imgUrlExportFile string +var exportFile *os.File +var db *sql.DB +var r SeriesReader + +type WrappedSeries struct { + Series series.Series + Image string +} + +type SeriesReader interface { + Read(path string) WrappedSeries + ID() int +} + +func GetSeriesReader(rType string) SeriesReader { + switch rType { + case "ACDv2": + return &ACDv2Reader{} + } + return nil +} + +func LogErr(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", err) + } +} + +func TryGetExistingSeries(s WrappedSeries, idmap map[int]int) int { + dbID, _ := idmap[s.Series.ID] + return dbID +} + +func InsertSeries(s WrappedSeries) int { + var dbID int + row := db.QueryRow("INSERT INTO grilist.appearances (id) VALUES (DEFAULT) RETURNING id;") + row.Scan(&dbID) + fmt.Printf( + "Assigned %d from %s to %d\n", + s.Series.ID, + sourceType, + dbID) + _, err := db.Exec("INSERT INTO grilist.appearances_id_mappings (appearance_id, source, source_id) VALUES ($1, $2, $3);", + dbID, r.ID(), s.Series.ID) + LogErr(err) + if s.Series.Name != "" { + _, err := db.Exec("INSERT INTO grilist.appearance_names (appearance_id, name, name_type) VALUES ($1, $2, $3);", + dbID, s.Series.Name, 4) + LogErr(err) + } + if s.Series.KanjiName != "" { + _, err := db.Exec("INSERT INTO grilist.appearance_names (appearance_id, name, name_type) VALUES ($1, $2, $3);", + dbID, s.Series.Name, 0) + LogErr(err) + } + if s.Series.RomajiName != "" { + _, err := db.Exec("INSERT INTO grilist.appearance_names (appearance_id, name, name_type) VALUES ($1, $2, $3);", + dbID, s.Series.Name, 1) + LogErr(err) + } + if s.Series.FuriganaName != "" { + _, err := db.Exec("INSERT INTO grilist.appearance_names (appearance_id, name, name_type) VALUES ($1, $2, $3);", + dbID, s.Series.Name, 3) + LogErr(err) + } + for _, v := range s.Series.OtherNames { + _, err := db.Exec("INSERT INTO grilist.appearance_names (appearance_id, name, name_type) VALUES ($1, $2, $3);", + dbID, v, 2) + LogErr(err) + } + if exportFile != nil { + if s.Image != "" { + fmt.Fprintf( + exportFile, + "%d %s\n", + dbID, + s.Image) + } + } + return dbID +} + +func InsertTags(dbID int, s WrappedSeries, taglist map[string]int) { + for _, v := range s.Series.Tags { + if _, ok := taglist[v]; ok { + continue + } + var id int + row := db.QueryRow("INSERT INTO grilist.tags (name, type) VALUES ($1, $2) RETURNING id;", + v, tags.TagTypeSeries) + row.Scan(&id) + taglist[v] = id + fmt.Printf("Inserted tag %s as %d\n", v, id) + } + rows, err := db.Query("SELECT tag_id FROM appearances_tags WHERE appearance_id = $1;", dbID) + LogErr(err) + existingTags := make(map[int]bool, 0) + for rows.Next() { + var tagID int + rows.Scan(&tagID) + existingTags[tagID] = true + } + for _, v := range s.Series.Tags { + tagID := taglist[v] + if _, ok := existingTags[tagID]; ok { + continue + } + _, err := db.Exec("INSERT INTO grilist.appearances_tags (appearance_id, tag_id) VALUES ($1, $2);", + dbID, tagID) + existingTags[tagID] = true + fmt.Printf("Assigned tag %s to %s\n", v, s.Series.Name) + LogErr(err) + } +} + +func main() { + flag.StringVar(&path, "path", "", "path of the source files") + flag.StringVar(&sourceType, "type", "", "type of the files (ACDv2)") + flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!") + flag.Parse() + if imgUrlExportFile != "" { + var err error + if exportFile, err = os.Create(imgUrlExportFile); err != nil { + fmt.Fprintf(os.Stderr, "%s\n", "error while accessing \"%s\"") + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + } + if path == "" { + fmt.Fprintf(os.Stderr, "%s\n", "\"path\" must be specified!") + flag.Usage() + os.Exit(1) + } + r = GetSeriesReader(sourceType) + if r == nil { + fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid series source!") + flag.Usage() + os.Exit(1) + } + config := grilist.LoadConfig() + var err error + db, err = sql.Open("postgres", config.DBConnectionString()) + if err == nil { + err = db.Ping() + } + if err != nil { + fmt.Fprintf(os.Stderr, "%s\n", "error with database connection") + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } + + taglist := make(map[string]int) + rows, err := db.Query("SELECT id, name FROM grilist.tags WHERE type = $1;", tags.TagTypeSeries) + LogErr(err) + for rows.Next() { + var id int + var name string + rows.Scan(&id, &name) + taglist[name] = id + } + idmap := make(map[int]int) + rows, err = db.Query("SELECT source_id, appearance_id FROM grilist.appearances_id_mappings WHERE source = $1;", r.ID()) + LogErr(err) + for rows.Next() { + var sourceID int + var appearanceID int + rows.Scan(&sourceID, &appearanceID) + idmap[sourceID] = appearanceID + } + + filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if info.IsDir() { + return nil + } + s := r.Read(path) + + var dbID int + if dbID = TryGetExistingSeries(s, idmap); dbID == 0 { + dbID = InsertSeries(s) + } + InsertTags(dbID, s, taglist) + + return nil + }) +} -- cgit v0.10.1