aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorjan <jan@ruken.pw>2016-10-01 17:38:00 (UTC)
committerjan <jan@ruken.pw>2016-10-01 17:38:00 (UTC)
commit6a130458c55de738ecced00989c74cc7125e17fe (patch)
treee46b4abfa2eb6814b1750d7cd430f719e7ab1965
parentf6fcd9bd7c17b1ac09e34a64c2545bc5a59da0a4 (diff)
jetzt warten wir auch
-rw-r--r--src/main.rs73
1 files changed, 45 insertions, 28 deletions
diff --git a/src/main.rs b/src/main.rs
index cb8b02d..8a3787b 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -13,7 +13,9 @@ use std::fs::File;
13use std::env; 13use std::env;
14use std::path::Path; 14use std::path::Path;
15use std::sync::{Arc, Mutex}; 15use std::sync::{Arc, Mutex};
16use std::sync::atomic::{AtomicUsize, Ordering};
16use std::thread; 17use std::thread;
18use std::time;
17 19
18mod pre_process; 20mod pre_process;
19mod section; 21mod section;
@@ -31,44 +33,59 @@ fn main() {
31 let raw_files = env::var("RAW_FILES").unwrap_or("characters".into()); 33 let raw_files = env::var("RAW_FILES").unwrap_or("characters".into());
32 let out_files = env::var("OUT").unwrap_or("json".into()); 34 let out_files = env::var("OUT").unwrap_or("json".into());
33 let base_path = Path::new(&raw_files); 35 let base_path = Path::new(&raw_files);
34 let out_path = Path::new(&out_files);
35 36
37 let active_threads = Arc::new(AtomicUsize::new(0));
36 38
37 39
38 let files: Arc<Mutex<Vec<DirEntry>>> = Arc::new(Mutex::new(WalkDir::new(base_path).min_depth(1).into_iter().filter_map(|e| e.ok()).collect())); 40 let files: Arc<Mutex<Vec<DirEntry>>> = Arc::new(Mutex::new(WalkDir::new(base_path).min_depth(1).into_iter().filter_map(|e| e.ok()).collect()));
39 41
40 for 1..MAX_THREADS { 42 for i in 1..MAX_THREADS {
41 43 let files = files.clone();
42 std::thread(move || { 44 let out_files = out_files.clone();
43 let entry: Option<DirEntry> = None; 45 active_threads.fetch_add(1, Ordering::SeqCst);
44 { 46 let active_threads = active_threads.clone();
45 entry = files.lock().unwrap().pop(); 47 thread::spawn(move || {
46 48 let out_path = Path::new(&out_files);
47 if entry.is_none() { 49 loop {
48 println!("thread finished"); 50 let entry: Option<DirEntry>;
49 return; 51 {
52 entry = files.lock().unwrap().pop();
53
54 if entry.is_none() {
55 break;
56 }
57 }
58 let entry = entry.unwrap();
59
60 let mut f = File::open(entry.path()).expect("could not open file");
61 let mut buf = String::new();
62 if let Err(_) = f.read_to_string(&mut buf) {
63 println!("invalid file: {}", entry.path().to_str().unwrap());
64 continue;
50 } 65 }
51 }
52 let entry = entry.unwrap();
53
54 let mut f = File::open(entry.path()).expect("could not open file");
55 let mut buf = String::new();
56 if let Err(_) = f.read_to_string(&mut buf) {
57 println!("invalid file: {}", entry.path().to_str().unwrap());
58 continue;
59 }
60 66
61 let buf = pre_process::strip_irrelevant_content(&buf); 67 let buf = pre_process::strip_irrelevant_content(&buf);
62 68
63 let mut char = Character::new(); 69 let mut char = Character::new();
64 char.parse(&buf); 70 char.parse(&buf);
65 71
66 let json = serde_json::to_string(&char).unwrap(); 72 let json = serde_json::to_string(&char).unwrap();
67 73
68 let out_file = out_path.join(entry.file_name().to_str().unwrap().replace("html", "json")); 74 let out_file = out_path.join(entry.file_name().to_str().unwrap().replace("html", "json"));
69 let mut o = File::create(&out_file).unwrap(); 75 let mut o = File::create(&out_file).unwrap();
70 o.write_all(json.as_bytes()).unwrap(); 76 o.write_all(json.as_bytes()).unwrap();
71 println!("{:?}", out_file); 77 println!("{:?}", out_file);
78 }
79 println!("thread {} finished", i);
80 active_threads.fetch_sub(1, Ordering::SeqCst);
72 }); 81 });
73 } 82 }
83
84 loop {
85 if active_threads.load(Ordering::SeqCst) == 0 {
86 println!("all threads finished.");
87 break;
88 }
89 thread::sleep(time::Duration::from_millis(100));
90 }
74} 91}