Grilimporter hinzugefuegt

2 files changed, 233 insertions, 0 deletions

diff --git a/tools/importer/ACDReader.go b/tools/importer/ACDReader.go
new file mode 100644
index 0000000..694087e
--- /dev/null
+++ b/tools/importer/ACDReader.go
@@ -0,0 +1,81 @@
+package main
+
+import (
+        "encoding/json"
+        "fmt"
+        "io/ioutil"
+        "os"
+        "regexp"
+        "strconv"
+        "strings"
+
+        "fagott.pw/grilist/modules/grils"
+)
+
+var tagRegex = regexp.MustCompile(`\<a.*?\>(.+)\<\/a\>`)
+
+type ACDReader struct{}
+
+func ifErrExit(err error) {
+        if err != nil {
+                fmt.Fprintf(os.Stderr, "%v\n", err)
+                os.Exit(1)
+        }
+}
+
+func toInt(s string) int {
+        i, err := strconv.ParseInt(s, 10, 32)
+        ifErrExit(err)
+        return int(i)
+}
+
+func (r *ACDReader) Read(path string) WrappedGril {
+        g := WrappedGril{}
+        g.Gril = grils.Gril{}
+        data, err := ioutil.ReadFile(path)
+        ifErrExit(err)
+        var jObj interface{}
+        err = json.Unmarshal(data, &jObj)
+        ifErrExit(err)
+        for k, v := range jObj.(map[string]interface{}) {
+                switch k {
+                case "Character ID":
+                        g.Gril.ID = toInt(v.(string))
+                case "Romaji Name":
+                        g.Gril.RomajiName = v.(string)
+                case "Japanese Name":
+                        g.Gril.KanjiName = v.(string)
+                case "Aliases":
+                        g.Gril.OtherNames = make([]string, 0)
+                        for _, n := range strings.Split(v.(string), ",") {
+                                if n == "" {
+                                        continue
+                                }
+                                g.Gril.OtherNames = append(
+                                        g.Gril.OtherNames,
+                                        strings.TrimSpace(n))
+                        }
+                case "Tagged":
+                        g.Gril.Tags = make([]string, 0)
+                        for _, n := range strings.Split(v.(string), ",") {
+                                if n == "" {
+                                        continue
+                                }
+                                tag := strings.TrimSpace(n)
+                                tag = tagRegex.FindStringSubmatch(tag)[1]
+                                g.Gril.Tags = append(
+                                        g.Gril.Tags,
+                                        tag)
+                        }
+                case "__img":
+                        g.Image = v.(string)
+                case "__thumb":
+                        g.Thumb = v.(string)
+                }
+        }
+        return g
+}
+
+func (r *ACDReader) ID() int {
+        return int(grils.DataSourceACD)
+}
diff --git a/tools/importer/main.go b/tools/importer/main.go
new file mode 100644
index 0000000..41733a1
--- /dev/null
+++ b/tools/importer/main.go
@@ -0,0 +1,152 @@
+package main
+
+import (
+        "database/sql"
+        "flag"
+        "fmt"
+        "os"
+        "path/filepath"
+
+        "fagott.pw/grilist/grilist"
+        "fagott.pw/grilist/modules/grils"
+
+        _ "github.com/lib/pq"
+)
+
+type WrappedGril struct {
+        Gril  grils.Gril
+        Image string
+        Thumb string
+}
+
+type GrilReader interface {
+        Read(path string) WrappedGril
+        ID() int
+}
+
+func GetGrilReader(rType string) GrilReader {
+        switch rType {
+        case "ACD":
+                return &ACDReader{}
+        }
+        return nil
+}
+
+func main() {
+        var path string
+        var sourceType string
+        var imgUrlExportFile string
+        flag.StringVar(&path, "path", "", "path of the source files")
+        flag.StringVar(&sourceType, "type", "", "type of the files (ACD, AniDB)")
+        flag.StringVar(&imgUrlExportFile, "imageexportfile", "", "image URLs will be exported to this file, THIS WILL OVERWRITE EVERYTHING IN THE FILE!")
+        flag.Parse()
+        var exportFile *os.File
+        if imgUrlExportFile != "" {
+                var err error
+                if exportFile, err = os.Create(imgUrlExportFile); err != nil {
+                        fmt.Fprintf(os.Stderr, "%s\n", "error while accessing \"%s\"")
+                        fmt.Fprintf(os.Stderr, "%v\n", err)
+                        os.Exit(1)
+                }
+        }
+        if path == "" {
+                fmt.Fprintf(os.Stderr, "%s\n", "\"path\" must be specified!")
+                flag.Usage()
+                os.Exit(1)
+        }
+        r := GetGrilReader(sourceType)
+        if r == nil {
+                fmt.Fprintf(os.Stderr, "%s\n", "\"type\" must be specified and a valid gril source!")
+                flag.Usage()
+                os.Exit(1)
+        }
+        config := grilist.LoadConfig()
+        db, err := sql.Open("postgres", config.DBConnectionString())
+        if err == nil {
+                err = db.Ping()
+        }
+        if err != nil {
+                fmt.Fprintf(os.Stderr, "%s\n", "error with database connection")
+                fmt.Fprintf(os.Stderr, "%v\n", err)
+                os.Exit(1)
+        }
+        taglist := make(map[string]int)
+        rows, _ := db.Query("SELECT id, name FROM grilist.tags;")
+        for rows.Next() {
+                var id int
+                var name string
+                rows.Scan(&id)
+                rows.Scan(&name)
+                taglist[name] = id
+        }
+        idmap := make(map[int]int)
+        rows, _ = db.Query("SELECT source_id, gril_id FROM grilist.grils_id_mappings WHERE source = $1;", r.ID())
+        for rows.Next() {
+                var sourceID int
+                var grilID int
+                rows.Scan(&sourceID)
+                rows.Scan(&grilID)
+                idmap[sourceID] = grilID
+        }
+        filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+                if info.IsDir() {
+                        return nil
+                }
+                g := r.Read(path)
+                for _, v := range g.Gril.Tags {
+                        if _, ok := taglist[v]; ok {
+                                continue
+                        }
+                        var id int
+                        row := db.QueryRow("INSERT INTO grilist.tags (name) VALUES ($1) RETURNING id;", v)
+                        row.Scan(&id)
+                        taglist[v] = id
+                }
+                if _, ok := idmap[g.Gril.ID]; !ok {
+                        row := db.QueryRow("INSERT INTO grilist.grils (age) VALUES (NULL) RETURNING id;")
+                        var dbID int
+                        row.Scan(&dbID)
+                        for _, v := range g.Gril.Tags {
+                                tagID := taglist[v]
+                                db.Exec("INSERT INTO grilist.grils_tags (gril_id, tag_id) VALUES ($1, $2);", dbID, tagID)
+                        }
+                        db.Exec(`INSERT INTO grilist.grils_id_mappings (gril_id,
+                                source, source_id) VALUES ($1, $2, $3);`,
+                                dbID, r.ID(), g.Gril.ID)
+                        if g.Gril.KanjiName != "" {
+                                db.Exec(`INSERT INTO grilist.gril_names
+                                (gril_id, name, name_type) VALUES ($1, $2,
+                                $3);`, dbID, g.Gril.KanjiName, 0)
+                        }
+                        if g.Gril.RomajiName != "" {
+                                db.Exec(`INSERT INTO grilist.gril_names
+                                (gril_id, name, name_type) VALUES ($1, $2,
+                                $3);`, dbID, g.Gril.RomajiName, 1)
+                        }
+                        for _, v := range g.Gril.OtherNames {
+                                db.Exec(`INSERT INTO grilist.gril_names
+                                (gril_id, name, name_type) VALUES ($1, $2,
+                                $3);`, dbID, v, 2)
+                        }
+                        if exportFile != nil {
+                                if g.Image != "" {
+                                        fmt.Fprintf(
+                                                exportFile,
+                                                "%d %d %s\n",
+                                                dbID,
+                                                0,
+                                                g.Image)
+                                }
+                                if g.Thumb != "" {
+                                        fmt.Fprintf(
+                                                exportFile,
+                                                "%d %d %s\n",
+                                                dbID,
+                                                1,
+                                                g.Thumb)
+                                }
+                        }
+                }
+                return nil
+        })
+}