package main import ( "database/sql" "flag" "fmt" "os" "path/filepath" "fagott.pw/grilist/grilist" "fagott.pw/grilist/models" _ "github.com/lib/pq" ) var path string var sourceType string var imgUrlExportFile string var exportFile *os.File var db *sql.DB var r GrilReader type WrappedGril struct { Gril models.Gril Image string Thumb string } type GrilReader interface { Read(path string) WrappedGril ID() int } func GetGrilReader(rType string) GrilReader { switch rType { case "ACD": return &ACDReader{} case "ACDv2": return &ACDv2Reader{} } return nil } func LogErr(err error) { if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) } } func TryGetExistingGril(g WrappedGril, idmap map[int]int) int { dbID, _ := idmap[g.Gril.ID] return dbID } func InsertGril(g WrappedGril) int { var dbID int row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;") row.Scan(&dbID) fmt.Printf( "Assigned %d from %s to %d\n", g.Gril.ID, sourceType, dbID) db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id, source, source_id) VALUES ($1, $2, $3);`, dbID, r.ID(), g.Gril.ID) if g.Gril.KanjiName != "" { _, err := db.Exec(`INSERT INTO grilist.gril_names (gril_id, name, name_type) VALUES ($1, $2, $3);`, dbID, g.Gril.KanjiName, 0) LogErr(err) } if g.Gril.RomajiName != "" { _, err := db.Exec(`INSERT INTO grilist.gril_names (gril_id, name, name_type) VALUES ($1, $2, $3);`, dbID, g.Gril.RomajiName, 1) LogErr(err) } for _, v := range g.Gril.OtherNames { _, err := db.Exec(`INSERT INTO grilist.gril_names (gril_id, name, name_type) VALUES ($1, $2, $3);`, dbID, v, 2) LogErr(err) } if exportFile != nil { if g.Image != "" { fmt.Fprintf( exportFile, "%d %s\n", dbID, g.Image) } if g.Thumb != "" { fmt.Fprintf( exportFile, "%d %s\n", dbID, g.Thumb) } } fmt.Printf("Inserted %s\n", g.Gril.RomajiName) return dbID } func InsertTags(dbID int, g WrappedGril, taglist map[string]int) { for _, v := range g.Gril.Tags { v := filterTags(v) if _, ok := taglist[v]; ok || v == "" { continue } var id int row := db.QueryRow(`INSERT INTO grilist.tags (name, type) VALUES ($1, $2) RETURNING id;`, v, models.TagTypeGril) row.Scan(&id) taglist[v] = id fmt.Printf("Inserted tag %s as %d\n", v, id) } rows, err := db.Query("SELECT tag_id FROM grils_tags WHERE gril_id = $1;", dbID) LogErr(err) existingTags := make(map[int]bool, 0) for rows.Next() { var tagID int rows.Scan(&tagID) existingTags[tagID] = true } for _, v := range g.Gril.Tags { v = filterTags(v) if v == "" { continue } tagID := taglist[v] if _, ok := existingTags[tagID]; ok { continue } _, err := db.Exec(`INSERT INTO grilist.grils_tags (gril_id, tag_id) VALUES ($1, $2);`, dbID, tagID) existingTags[tagID] = true fmt.Printf("Assigned tag %s to %s\n", v, g.Gril.RomajiName) LogErr(err) } } func InsertTraits(dbID int, g WrappedGril, traitlist map[string]int) { for _, v := range g.Gril.Traits { v.Name = filterTraits(v.Name) if _, ok := traitlist[v.Name]; ok || v.Name == "" { continue } var id int row := db.QueryRow("INSERT INTO grilist.traits (name) VALUES ($1) RETURNING id;", v.Name) row.Scan(&id) traitlist[v.Name] = id fmt.Printf("Inserted trait %s as %d\n", v.Name, id) } rows, err := db.Query("SELECT trait_id FROM grils_traits WHERE gril_id = $1;", dbID) LogErr(err) existingTraits := make(map[int]bool, 0) for rows.Next() { var traitID int rows.Scan(&traitID) existingTraits[traitID] = true } for _, v := range g.Gril.Traits { v.Name = filterTraits(v.Name) if v.Name == "" { continue } traitID := traitlist[v.Name] if _, ok := existingTraits[traitID]; ok { continue } _, err := db.Exec(`INSERT INTO grilist.grils_traits (gril_id, trait_id, value, official) VALUES ($1, $2, $3, $4);`, dbID, traitID, v.Value, v.OfficialValue) existingTraits[traitID] = true fmt.Printf("Assigned trait %s to %s\n", v, g.Gril.RomajiName) LogErr(err) } } func InsertSeries(dbID int, g WrappedGril, seriesmap map[int]int) { rows, err := db.Query("SELECT appearance_id FROM grils_appearances WHERE gril_id = $1;", dbID) LogErr(err) existingSeries := make(map[int]bool, 0) for rows.Next() { var seriesID int rows.Scan(&seriesID) existingSeries[seriesID] = true } for _, v := range g.Gril.Series { seriesID, ok := seriesmap[v.ID] if !ok { panic(fmt.Sprintf("Could not find series %s! Did you import the series?", v.ID)) } if _, ok := existingSeries[seriesID]; ok { continue } _, err := db.Exec("INSERT INTO grilist.grils_appearances (gril_id, appearance_id) VALUES ($1, $2);", dbID, seriesID) existingSeries[seriesID] = true fmt.Printf("Assigned series %d to %s\n", seriesID, g.Gril.RomajiName) LogErr(err) } } func main() { flag.StringVar(&path, "path", "", "path of the source files") flag.StringVar(&sourceType, "type", "", "type of the files (ACD, AniDB)") flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!") flag.Parse() if imgUrlExportFile != "" { var err error if exportFile, err = os.Create(imgUrlExportFile); err != nil { fmt.Fprintf(os.Stderr, "%s\n", "error while accessing \"%s\"") fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } } if path == "" { fmt.Fprintf(os.Stderr, "%s\n", "\"path\" must be specified!") flag.Usage() os.Exit(1) } r = GetGrilReader(sourceType) if r == nil { fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid gril source!") flag.Usage() os.Exit(1) } config := grilist.LoadConfig() var err error db, err = sql.Open("postgres", config.DBConnectionString()) if err == nil { err = db.Ping() } if err != nil { fmt.Fprintf(os.Stderr, "%s\n", "error with database connection") fmt.Fprintf(os.Stderr, "%v\n", err) os.Exit(1) } taglist := make(map[string]int) rows, _ := db.Query("SELECT id, name FROM grilist.tags WHERE type = $1;", models.TagTypeGril) for rows.Next() { var id int var name string rows.Scan(&id, &name) taglist[name] = id } traitlist := make(map[string]int) rows, _ = db.Query("SELECT id, name FROM grilist.traits;") for rows.Next() { var id int var name string rows.Scan(&id, &name) traitlist[name] = id } idmap := make(map[int]int) rows, _ = db.Query("SELECT source_id, gril_id FROM grilist.grils_id_mappings WHERE source = $1;", r.ID()) for rows.Next() { var sourceID int var grilID int rows.Scan(&sourceID, &grilID) idmap[sourceID] = grilID } seriesmap := make(map[int]int) rows, _ = db.Query("SELECT appearance_id, source_id FROM grilist.appearances_id_mappings WHERE source = $1", r.ID()) for rows.Next() { var seriesID int var sourceID int rows.Scan(&seriesID, &sourceID) seriesmap[sourceID] = seriesID } filepath.Walk(path, func(path string, info os.FileInfo, err error) error { if info.IsDir() { return nil } g := r.Read(path) var dbID int if dbID = TryGetExistingGril(g, idmap); dbID == 0 { dbID = InsertGril(g) } InsertTags(dbID, g, taglist) InsertTraits(dbID, g, traitlist) InsertSeries(dbID, g, seriesmap) return nil }) }