WARP 1.0

- Added FFI
- Added a sidebar to the UI
- Added project, directory and archive processing
- Added generic `Container` interface for extensible stores of WARP data
- Fixed type references being constructed and pulled incorrectly
- Added HTML, Markdown and JSON report generation
- Made the WARP information added as an analysis activity
- Flattened the signatures directory, the target information is stored in the file now
- Matched function information is stored as function metadata in the database to reliably persist, alongside the function GUID
- Split the matching out from the application, allowing you to match on a given function without applying it
- Added more/better tests
- Added support for binaries with multiple architectures, the functions are now also queried based off the Target, see WARP spec for more details
- Greatly improved support for RISC architectures, see WARP spec for more details
- Greatly improved UX when loading files after the fact, will now sanely rerun the matcher
- Omitted the function type if not a user type, this greatly reduces file size
- Improved support for functions that reference a page aligned base pointer, see WARP spec for more details
- Removed some extra cache structures that were causing erroneous behavior
- Fixed edge-case in LLIL traversal missing some constant pointers, this was a bug in the Rust bindings
- Added support for function comments
- Made long running tasks, such as generating, matching and loading signatures, cancellable where possible
- Made function constraints more versatile, allowing for easy extensions in the future, see WARP spec for details
- Added options to signature generation, such as what data to store, and whether to compress the data or not
- Made all long running tasks prompt the user for required information before the task starts, allowing users to "set it and forget it" and not have to baby sit the finalization of the task
- Myriad of other changes to the actual WARP format that impact performance, file size and general feature set, see https://github.com/Vector35/warp for more details

3 files changed, 77 insertions, 65 deletions

diff --git a/plugins/warp/benches/convert.rs b/plugins/warp/benches/convert.rs
index 7d7f40b3..e0acdc15 100644
--- a/plugins/warp/benches/convert.rs
+++ b/plugins/warp/benches/convert.rs
@@ -4,33 +4,36 @@ use criterion::{criterion_group, criterion_main, Criterion};
 use std::path::PathBuf;
 use warp_ninja::convert::from_bn_type;
 
+// These are the target files present in OUT_DIR
+// Add the files to fixtures/bin
+static TARGET_FILES: [&str; 1] = ["atox.obj"];
+
 pub fn type_conversion_benchmark(c: &mut Criterion) {
     let session = Session::new().expect("Failed to initialize session");
     let out_dir = env!("OUT_DIR").parse::<PathBuf>().unwrap();
-    for entry in std::fs::read_dir(out_dir).expect("Failed to read OUT_DIR") {
-        let entry = entry.expect("Failed to read directory entry");
-        let path = entry.path();
-        if path.is_file() {
-            if let Some(bv) = session.load(path.to_str().unwrap()) {
-                let functions = bv.functions();
-                c.bench_function("type conversion all functions", |b| {
-                    b.iter(|| {
-                        for func in &functions {
-                            from_bn_type(&bv, &func.function_type(), u8::MAX);
-                        }
-                    })
-                });
+    for file_name in TARGET_FILES {
+        let path = out_dir.join(file_name);
+        let view = session.load(&path).expect("Failed to load view");
+
+        // Bench function types.
+        let functions: Vec<_> = view.functions().iter().map(|f| f.to_owned()).collect();
+        c.bench_function("type conversion all functions", |b| {
+            b.iter(|| {
+                for func in &functions {
+                    from_bn_type(&view, &func.function_type(), u8::MAX);
+                }
+            })
+        });
 
-                let types = bv.types();
-                c.bench_function("type conversion all types", |b| {
-                    b.iter(|| {
-                        for ty in &types {
-                            from_bn_type(&bv, &ty.ty, u8::MAX);
-                        }
-                    })
-                });
-            }
-        }
+        // Bench view types.
+        let types = view.types().to_vec();
+        c.bench_function("type conversion all types", |b| {
+            b.iter(|| {
+                for ty in &types {
+                    from_bn_type(&view, &ty.ty, u8::MAX);
+                }
+            })
+        });
     }
 }
 
diff --git a/plugins/warp/benches/function.rs b/plugins/warp/benches/function.rs
index 0ddf9ce2..dc5016be 100644
--- a/plugins/warp/benches/function.rs
+++ b/plugins/warp/benches/function.rs
@@ -2,43 +2,43 @@ use binaryninja::binary_view::BinaryViewExt;
 use binaryninja::headless::Session;
 use criterion::{criterion_group, criterion_main, Criterion};
 use rayon::prelude::*;
+use std::path::PathBuf;
 use warp_ninja::build_function;
-use warp_ninja::cache::FunctionCache;
+
+// These are the target files present in OUT_DIR
+// Add the files to fixtures/bin
+static TARGET_FILES: [&str; 1] = ["atox.obj"];
 
 pub fn function_benchmark(c: &mut Criterion) {
     let session = Session::new().expect("Failed to initialize session");
-    let bv = session.load(env!("TEST_BIN_LIBRARY_OBJ")).unwrap();
-    let functions = bv.functions();
-    assert_eq!(functions.len(), 6);
-    let mut function_iter = functions.into_iter();
-    let first_function = function_iter.next().unwrap();
-
-    c.bench_function("signature first function", |b| {
-        b.iter(|| {
-            let _ = build_function(&first_function, &first_function.low_level_il().unwrap());
-        })
-    });
+    let out_dir = env!("OUT_DIR").parse::<PathBuf>().unwrap();
+    for file_name in TARGET_FILES {
+        let path = out_dir.join(file_name);
+        let view = session.load(&path).expect("Failed to load view");
 
-    c.bench_function("signature all functions", |b| {
-        b.iter(|| {
-            for func in &functions {
-                let _ = build_function(&func, &func.low_level_il().unwrap());
-            }
-        })
-    });
+        let functions: Vec<_> = view.functions().iter().map(|f| f.to_owned()).collect();
+        // Bench all functions sequentially
+        c.bench_function("signature all functions", |b| {
+            b.iter(|| {
+                for func in &functions {
+                    let _ = build_function(&func, &func.lifted_il().unwrap());
+                }
+            })
+        });
 
-    let cache = FunctionCache::default();
-    c.bench_function("signature all functions rayon", |b| {
-        b.iter(|| {
-            functions
-                .par_iter()
-                .map_with(cache.clone(), |par_cache, func| {
-                    let llil = func.low_level_il().ok()?;
-                    Some(par_cache.function(func.as_ref(), llil.as_ref()))
-                })
-                .collect::<Vec<_>>()
-        })
-    });
+        // Bench all functions in parallel
+        c.bench_function("signature all functions rayon", |b| {
+            b.iter(|| {
+                functions
+                    .par_iter()
+                    .filter_map(|func| {
+                        let llil = func.lifted_il().ok()?;
+                        Some(build_function(func.as_ref(), llil.as_ref()))
+                    })
+                    .collect::<Vec<_>>()
+            })
+        });
+    }
 }
 
 criterion_group!(benches, function_benchmark);
diff --git a/plugins/warp/benches/guid.rs b/plugins/warp/benches/guid.rs
index 046886b5..eddf2f06 100644
--- a/plugins/warp/benches/guid.rs
+++ b/plugins/warp/benches/guid.rs
@@ -1,22 +1,31 @@
 use binaryninja::binary_view::BinaryViewExt;
 use binaryninja::headless::Session;
 use criterion::{criterion_group, criterion_main, Criterion};
+use std::path::PathBuf;
 use warp_ninja::function_guid;
 
+// These are the target files present in OUT_DIR
+// Add the files to fixtures/bin
+static TARGET_FILES: [&str; 1] = ["atox.obj"];
+
 pub fn guid_benchmark(c: &mut Criterion) {
     let session = Session::new().expect("Failed to initialize session");
-    let bv = session.load(env!("TEST_BIN_LIBRARY_OBJ")).unwrap();
-    let functions = bv.functions();
-    assert_eq!(functions.len(), 6);
-    let mut function_iter = functions.into_iter();
-    let first_function = function_iter.next().unwrap();
+    let out_dir = env!("OUT_DIR").parse::<PathBuf>().unwrap();
+    for file_name in TARGET_FILES {
+        let path = out_dir.join(file_name);
+        let view = session.load(&path).expect("Failed to load view");
 
-    c.bench_function("function guid", |b| {
-        b.iter(|| {
-            let llil = first_function.low_level_il().unwrap();
-            function_guid(&first_function, &llil);
-        })
-    });
+        let functions: Vec<_> = view.functions().iter().map(|f| f.to_owned()).collect();
+        // Bench all functions sequentially
+        c.bench_function("guid all functions", |b| {
+            b.iter(|| {
+                for func in &functions {
+                    let llil = func.lifted_il().unwrap();
+                    function_guid(&func, &llil);
+                }
+            })
+        });
+    }
 }
 
 criterion_group!(benches, guid_benchmark);