From 6374e3fecfda083cdb7c2b454e5ed58cd631cc20 Mon Sep 17 00:00:00 2001
From: Mason Reed <mason@vector35.com>
Date: Wed, 6 Nov 2024 10:58:03 -0500
Subject: WARP: Parallelize sigem `data_from_directory`

---
 plugins/warp/src/bin/sigem.rs | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'plugins')

diff --git a/plugins/warp/src/bin/sigem.rs b/plugins/warp/src/bin/sigem.rs
index f752487d..9d158cf3 100644
--- a/plugins/warp/src/bin/sigem.rs
+++ b/plugins/warp/src/bin/sigem.rs
@@ -130,18 +130,26 @@ fn data_from_archive<R: Read>(mut archive: Archive<R>) -> Option<Data> {
 }
 
 fn data_from_directory(dir: PathBuf) -> Option<Data> {
-    let unmerged_data = WalkDir::new(dir)
+    let files = WalkDir::new(dir)
         .into_iter()
         .filter_map(|e| {
             let path = e.ok()?.into_path();
             if path.is_file() {
-                log::info!("Creating data for FILE {:?}...", path);
-                data_from_file(&path)
+                Some(path)
             } else {
                 None
             }
         })
         .collect::<Vec<_>>();
+
+    let unmerged_data = files
+        .into_par_iter()
+        .filter_map(|path| {
+            log::debug!("Creating data for FILE {:?}...", path);
+            data_from_file(&path)
+        })
+        .collect::<Vec<_>>();
+
     if !unmerged_data.is_empty() {
         Some(Data::merge(&unmerged_data))
     } else {
-- 
cgit v1.3.1