7 files changed, 104 insertions, 40 deletions

diff --git a/src/character.rs b/src/character.rs
index 0630a45..f4686cf 100644
--- a/src/character.rs
+++ b/src/character.rs
@@ -42,19 +42,29 @@ pub struct Character {
 
 impl Names {
     pub fn new() -> Self {
-        Names { romaji: String::new(), japanese: String::new(), aliases: vec![] }
+        Names {
+            romaji: String::new(),
+            japanese: String::new(),
+            aliases: vec![],
+        }
     }
 }
 
 impl Images {
     pub fn new() -> Self {
-        Images { thumb: String::new(), full: String::new() }
+        Images {
+            thumb: String::new(),
+            full: String::new(),
+        }
     }
 }
 
 impl Traits {
     pub fn new() -> Self {
-        Traits { official: vec![], indexed: vec![] }
+        Traits {
+            official: vec![],
+            indexed: vec![],
+        }
     }
 }
 
@@ -67,13 +77,14 @@ impl Character {
             traits: Traits::new(),
             assignments: vec![],
             extra: vec![],
-            role: None
+            role: None,
         }
     }
 
     pub fn parse(&mut self, buf: &str) {
-        let re_extras = Regex::new(r#"(?is)Extra Details \| [0-9]+</H3>.*?<dl>(.*?)</dl>"#).unwrap();
-        let mut sections = get_sections();    
+        let re_extras = Regex::new(r#"(?is)Extra Details \| [0-9]+</H3>.*?<dl>(.*?)</dl>"#)
+                            .unwrap();
+        let mut sections = get_sections();
         section::process(&buf, &mut sections);
 
         let caps = re_extras.captures(&buf);
@@ -89,40 +100,57 @@ impl Character {
 
             self.name.romaji = name.data["romaji".into()].clone();
             self.name.japanese = name.data["japanese".into()].clone();
-            
+
             if name.data["aliases".into()].len() > 0 {
-                self.name.aliases = name.data["aliases".into()].split(", ").map(|s| s.to_string()).collect();
+                self.name.aliases = name.data["aliases".into()]
+                                        .split(", ")
+                                        .map(|s| s.to_string())
+                                        .collect();
             }
-            
+
             self.image.thumb = image.data["thumb".into()].clone();
             self.image.full = image.data["full".into()].clone();
-            
+
             if tags.data.contains_key("tags_raw".into()) {
                 self.tags = tags::parse(&tags.data["tags_raw".into()]);
             }
 
-            self.traits.official = dl_list::parse(&(&sections["traits"] as &Section).data["official_raw".into()]);
-            self.traits.indexed = dl_list::parse(&(&sections["traits"] as &Section).data["indexed_raw".into()]);
+            self.traits.official =
+                dl_list::parse(&(&sections["traits"] as &Section).data["official_raw".into()]);
+            self.traits.indexed =
+                dl_list::parse(&(&sections["traits"] as &Section).data["indexed_raw".into()]);
 
             self.assignments = tiles::parse_tile_link_ids(&(&sections["assignments"] as &Section).data["raw".into()], "series");
-            
+
             if misc.data["role".into()].len() > 0 {
                 self.role = Some(misc.data["role".into()].clone());
             }
         }
-   }
+    }
 }
 
 fn get_sections() -> HashMap<String, Section> {
     let mut s: HashMap<String, Section> = HashMap::new();
 
     s.insert("name".into(), Section::new("name", r#"(?is)Romaji Name.*?<TD>(.*?)\s?</TD>.*?Japanese Name.*?<TD>(.*?)\s?</TD>.*?Aliases.*?<TD>(.*?)\s?</TD>"#, vec!["romaji", "japanese", "aliases"]));
-    s.insert("misc".into(), Section::new("misc", r#"(?is)Role</TH>.*?<TD>(.*?)\s?</TD>"#, vec!["role"]));
+    s.insert("misc".into(),
+             Section::new("misc",
+                          r#"(?is)Role</TH>.*?<TD>(.*?)\s?</TD>"#,
+                          vec!["role"]));
     s.insert("image".into(), Section::new("image", r#"(?is)<H3 id="section99">.*<img src="(.*?)" alt=.*?></a><p><a href="(.*?)">View Full Size Image"#, vec!["thumb", "full"]));
-    s.insert("tags".into(), Section::new("tags", r#"(?is)tagged as</P>.*?<TH>(.*?)</TH>"#, vec!["tags_raw"]));
+    s.insert("tags".into(),
+             Section::new("tags",
+                          r#"(?is)tagged as</P>.*?<TH>(.*?)</TH>"#,
+                          vec!["tags_raw"]));
     s.insert("traits".into(), Section::new("traits", r#"(?is)indexed traits</P>.*?<dl>(.*?)</dl>.*?official traits\s?</P>.*?<dl>(.*?)</dl>"#, vec!["indexed_raw", "official_raw"]));
-    s.insert("assignments".into(), Section::new("assignments", r#"(?is)appears in the following</P>(.*?)</UL>"#, vec!["raw"]));
-    s.insert("chars_similar_traits".into(), Section::new("assignments", r#"(?is)with Similar Traits</H3>(.*?)</UL>"#, vec!["raw"]));
+    s.insert("assignments".into(),
+             Section::new("assignments",
+                          r#"(?is)appears in the following</P>(.*?)</UL>"#,
+                          vec!["raw"]));
+    s.insert("chars_similar_traits".into(),
+             Section::new("assignments",
+                          r#"(?is)with Similar Traits</H3>(.*?)</UL>"#,
+                          vec!["raw"]));
 
     s
 }
diff --git a/src/dl_list.rs b/src/dl_list.rs
index 3327388..17d49fc 100644
--- a/src/dl_list.rs
+++ b/src/dl_list.rs
@@ -9,5 +9,12 @@ pub struct DLListItem {
 pub fn parse(s: &str) -> Vec<DLListItem> {
     let reg_list_item = Regex::new(r#"(?is)<dt.*?>(.*?)</dt>.*?<dd>(.*?)</dd>"#).unwrap();
 
-    reg_list_item.captures_iter(s).map(|c| DLListItem { name: c.at(1).unwrap().into(), value: c.at(2).unwrap().into() }).collect()
+    reg_list_item.captures_iter(s)
+                 .map(|c| {
+                     DLListItem {
+                         name: c.at(1).unwrap().into(),
+                         value: c.at(2).unwrap().into(),
+                     }
+                 })
+                 .collect()
 }
diff --git a/src/main.rs b/src/main.rs
index 7481376..f15a3dd 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,9 +1,9 @@
 #![feature(proc_macro)]
-#[macro_use] extern crate serde_derive;
+#[macro_use]
+extern crate serde_derive;
 extern crate serde_json;
 
 extern crate regex;
-extern crate yaml_rust;
 
 extern crate walkdir;
 use walkdir::{WalkDir, DirEntry};
@@ -38,8 +38,12 @@ fn main() {
 
     let active_threads = Arc::new(AtomicUsize::new(0));
 
-    
-    let files: Arc<Mutex<Vec<DirEntry>>> = Arc::new(Mutex::new(WalkDir::new(base_path).min_depth(1).into_iter().filter_map(|e| e.ok()).collect()));
+
+    let files: Arc<Mutex<Vec<DirEntry>>> = Arc::new(Mutex::new(WalkDir::new(base_path)
+                                                                   .min_depth(1)
+                                                                   .into_iter()
+                                                                   .filter_map(|e| e.ok())
+                                                                   .collect()));
 
     for i in 0..MAX_THREADS {
         let files = files.clone();
@@ -58,7 +62,7 @@ fn main() {
                     }
                 }
                 let entry = entry.unwrap();
-                
+
                 let mut f = File::open(entry.path()).expect("could not open file");
                 let mut buf = String::new();
                 if let Err(_) = f.read_to_string(&mut buf) {
@@ -68,12 +72,16 @@ fn main() {
 
                 let buf = pre_process::strip_irrelevant_content(&buf);
 
+                println!("{:?}", entry.path());
                 let mut char = Series::new();
                 char.parse(&buf);
 
                 let json = serde_json::to_string(&char).unwrap();
 
-                let out_file = out_path.join(entry.file_name().to_str().unwrap().replace("html", "json"));
+                let out_file = out_path.join(entry.file_name()
+                                                  .to_str()
+                                                  .unwrap()
+                                                  .replace("html", "json"));
                 let mut o = File::create(&out_file).unwrap();
                 o.write_all(json.as_bytes()).unwrap();
                 println!("{:?}", out_file);
diff --git a/src/pre_process.rs b/src/pre_process.rs
index 877ddc3..c68528f 100644
--- a/src/pre_process.rs
+++ b/src/pre_process.rs
@@ -1,15 +1,13 @@
 pub fn strip_irrelevant_content(s: &str) -> String {
     let mut retn = "";
-    match s.find(r#"<div class=profile id=profile>"#) {
+    match s.find(r#"<A href="watchdontwatch.php">"#) {
         Some(pos) => retn = &s[pos..],
         None => (),
     };
 
-    match s.find(r#"<INPUT style="font-size: 2em;" TYPE=SUBMIT NAME="votes" VALUE="Cast Votes">"#) {
+    match s.find(r#"<H3>Characters"#) {
         Some(pos) => retn = &s[..pos],
         None => (),
     };
     return retn.into();
 }
-
-
diff --git a/src/section.rs b/src/section.rs
index 7e492b1..48f237d 100644
--- a/src/section.rs
+++ b/src/section.rs
@@ -1,6 +1,7 @@
 use super::regex::Regex;
 use std::collections::HashMap;
 
+#[derive(Debug)]
 pub struct Section {
     pub name: String,
     pub re: Regex,
diff --git a/src/series.rs b/src/series.rs
index d111fb8..3037e6a 100644
--- a/src/series.rs
+++ b/src/series.rs
@@ -21,12 +21,17 @@ pub struct Names {
 #[derive(Debug, Serialize)]
 pub struct Series {
     pub name: Names,
+    pub tags: Vec<String>,
 }
 
 impl Names {
     pub fn new() -> Self {
         Names {
-            String::new(), String::new(), String::new(), String::new(), String::new()
+            english: String::new(),
+            aliases: String::new(),
+            romaji: String::new(),
+            furigana: String::new(),
+            japanese: String::new(),
         }
     }
 }
@@ -34,30 +39,40 @@ impl Names {
 impl Series {
     pub fn new() -> Self {
         Series {
-            name: Names::new()
+            name: Names::new(),
+            tags: vec![],
         }
     }
 
     pub fn parse(&mut self, buf: &str) {
-        let mut sections = get_sections();    
+        let mut sections = get_sections();
         section::process(&buf, &mut sections);
 
+        let re_genre_tags = Regex::new(r#"(?is)Genre Tags.*?>(.*?)</td>"#).unwrap();
+        let re_genre_tag = Regex::new(r#"[0-9]">(.*?)</A>"#).unwrap();
+
+        for cap in re_genre_tags.captures_iter(&buf) {
+            self.tags = re_genre_tag.captures_iter(cap.at(1).unwrap())
+                                    .map(|v| v.at(1).unwrap().to_string())
+                                    .collect();
+        }
+
         {
             let name: &Section = &sections["name".into()];
 
-            self.name.english = name["english".into()];
-            self.name.aliases = name["aliases".into()];
-            self.name.romaji = name["romaji".into()];
-            self.name.furigana = name["furigana".into()];
-            self.name.japanese = name["japanese".into()];
+            self.name.english = name.data["english".into()].to_string();
+            self.name.aliases = name.data["aliases".into()].to_string();
+            self.name.romaji = name.data["romaji".into()].to_string();
+            self.name.furigana = name.data["furigana".into()].to_string();
+            self.name.japanese = name.data["japanese".into()].to_string();
         }
-   }
+    }
 }
 
 fn get_sections() -> HashMap<String, Section> {
     let mut s: HashMap<String, Section> = HashMap::new();
 
-    s.insert("name".into(), Section::new("name", r#"(?is)English Title.*?<TD>(.*?)</TD>.*?Aliases.*?<TD>(.*?)?</TD>.*?Romaji Title.*?<TD.*?>(.*?)</TD>.*?Furigana Title.*?<TD.*?>(.*?)</TD>.*?Japanese Title.*?<TD.*?>(.*?)</TD>"#, vec!["english", "aliases", "romaji", "furigana", "japanese"]));
+    s.insert("name".into(), Section::new("name", r#"(?is)English Title.*?<TD>(.*?)<.*?Aliases.*?<TD>(.*?)<.*?Romaji Title.*?<TD.*?>(.*?)</.*?Furigana Title.*?<TD.*?>(.*?)</.*?Japanese Title.*?<TD.*?>(.*?)</"#, vec!["english", "aliases", "romaji", "furigana", "japanese"]));
 
     s
 }
diff --git a/src/tags.rs b/src/tags.rs
index 561c54d..63db847 100644
--- a/src/tags.rs
+++ b/src/tags.rs
@@ -11,5 +11,12 @@ pub struct Tag {
 pub fn parse(s: &str) -> Vec<Tag> {
     let reg_tag = Regex::new(r#"(?is)<a href="tags\.php\?id=([0-9]+)">(.*?)</a>"#).unwrap();
 
-    reg_tag.captures_iter(s).map(|c| Tag { id: u32::from_str(c.at(1).unwrap()).unwrap(), name: c.at(2).unwrap().into() }).collect()
+    reg_tag.captures_iter(s)
+           .map(|c| {
+               Tag {
+                   id: u32::from_str(c.at(1).unwrap()).unwrap(),
+                   name: c.at(2).unwrap().into(),
+               }
+           })
+           .collect()
 }