multithreading

1 files changed, 37 insertions, 16 deletions

diff --git a/src/main.rs b/src/main.rs
index c91861a..cb8b02d 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -6,12 +6,14 @@ extern crate regex;
 extern crate yaml_rust;
 
 extern crate walkdir;
-use walkdir::WalkDir;
+use walkdir::{WalkDir, DirEntry};
 
 use std::io::prelude::*;
 use std::fs::File;
 use std::env;
 use std::path::Path;
+use std::sync::{Arc, Mutex};
+use std::thread;
 
 mod pre_process;
 mod section;
@@ -23,6 +25,8 @@ mod tags;
 mod dl_list;
 mod tiles;
 
+static MAX_THREADS: u32 = 12;
+
 fn main() {
     let raw_files = env::var("RAW_FILES").unwrap_or("characters".into());
     let out_files = env::var("OUT").unwrap_or("json".into());
@@ -30,24 +34,41 @@ fn main() {
     let out_path = Path::new(&out_files);
 
 
-    for entry in WalkDir::new(base_path).min_depth(1).into_iter().filter_map(|e| e.ok()) {
-        let mut f = File::open(entry.path()).expect("could not open file");
-        let mut buf = String::new();
-        if let Err(_) = f.read_to_string(&mut buf) {
-            println!("invalid file: {}", entry.path().to_str().unwrap());
-            continue;
-        }
+    
+    let files: Arc<Mutex<Vec<DirEntry>>> = Arc::new(Mutex::new(WalkDir::new(base_path).min_depth(1).into_iter().filter_map(|e| e.ok()).collect()));
+
+    for 1..MAX_THREADS {
+
+        std::thread(move || {
+            let entry: Option<DirEntry> = None;
+            {
+                entry = files.lock().unwrap().pop();
+
+                if entry.is_none() {
+                    println!("thread finished");
+                    return;
+                }
+            }
+            let entry = entry.unwrap();
+            
+            let mut f = File::open(entry.path()).expect("could not open file");
+            let mut buf = String::new();
+            if let Err(_) = f.read_to_string(&mut buf) {
+                println!("invalid file: {}", entry.path().to_str().unwrap());
+                continue;
+            }
 
-        let buf = pre_process::strip_irrelevant_content(&buf);
+            let buf = pre_process::strip_irrelevant_content(&buf);
 
-        let mut char = Character::new();
-        char.parse(&buf);
+            let mut char = Character::new();
+            char.parse(&buf);
 
-        let json = serde_json::to_string(&char).unwrap();
+            let json = serde_json::to_string(&char).unwrap();
 
-        let out_file = out_path.join(entry.file_name().to_str().unwrap().replace("html", "json"));
-        let mut o = File::create(&out_file).unwrap();
-        o.write_all(json.as_bytes()).unwrap();
-        println!("{:?}", out_file);
+            let out_file = out_path.join(entry.file_name().to_str().unwrap().replace("html", "json"));
+            let mut o = File::create(&out_file).unwrap();
+            o.write_all(json.as_bytes()).unwrap();
+            println!("{:?}", out_file);
+        });
     }
 }