diff options
| -rw-r--r-- | Cargo.lock | 185 | ||||
| -rw-r--r-- | Cargo.toml | 2 | ||||
| -rw-r--r-- | plugins/warp/Cargo.toml | 2 | ||||
| -rw-r--r-- | plugins/warp/README.md | 41 | ||||
| -rw-r--r-- | plugins/warp/examples/headless/Cargo.toml | 14 | ||||
| -rw-r--r-- | plugins/warp/examples/headless/README.md | 44 | ||||
| -rw-r--r-- | plugins/warp/examples/headless/src/main.rs | 182 |
7 files changed, 427 insertions, 43 deletions
@@ -244,6 +244,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" [[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2", +] + +[[package]] name = "bon" version = "3.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -345,6 +354,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" [[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] name = "ciborium" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -474,6 +489,19 @@ dependencies = [ ] [[package]] +name = "console" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b430743a6eb14e9764d4260d4c0d8123087d504eeb9c48f2b2a5e810dd369df4" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.61.2", +] + +[[package]] name = "cpp_demangle" version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -556,6 +584,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] +name = "ctrlc" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73736a89c4aff73035ba2ed2e565061954da00d4970fc9ac25dcc85a2a20d790" +dependencies = [ + "dispatch2", + "nix", + "windows-sys 0.61.2", +] + +[[package]] name = "darling" version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -662,6 +701,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89a09f22a6c6069a18470eb92d2298acf25463f14256d24778e1230d789a2aec" dependencies = [ "bitflags 2.9.1", + "block2", + "libc", "objc2", ] @@ -752,6 +793,29 @@ dependencies = [ ] [[package]] +name = "env_filter" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c863f0904021b108aa8b2f55046443e6b1ebde8fd4a15c399893aae4fa069f" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + +[[package]] name = "equivalent" version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1080,12 +1144,25 @@ dependencies = [ ] [[package]] +name = "indicatif" +version = "0.18.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ade6dfcba0dfb62ad59e59e7241ec8912af34fd29e0e743e3db992bd278e8b65" +dependencies = [ + "console 0.16.1", + "portable-atomic", + "unicode-width", + "unit-prefix", + "web-time", +] + +[[package]] name = "insta" version = "1.43.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" dependencies = [ - "console", + "console 0.15.11", "once_cell", "serde", "similar", @@ -1137,6 +1214,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] +name = "jiff" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be1f93b8b1eb69c77f24bbb0afdf66f54b632ee39af40ca21c4365a1d7347e49" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde", +] + +[[package]] +name = "jiff-static" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03343451ff899767262ec32146f6d559dd759fdadf42ff0e227c7c48f72594b4" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] name = "jobserver" version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1314,6 +1415,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e91c22f111c205b788cc10e2a58aed039d5dd5554d3a9236be3d413a912ff3cb" [[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.9.1", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] name = "nom" version = "7.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1373,9 +1486,9 @@ dependencies = [ [[package]] name = "objc2" -version = "0.6.1" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c6597e14493ab2e44ce58f2fdecf095a51f12ca57bec060a11c57332520551" +checksum = "b7c2599ce0ec54857b29ce62166b0ed9b4f6f1a70ccc9a71165b6154caca8c05" dependencies = [ "objc2-encode", ] @@ -1618,6 +1731,21 @@ dependencies = [ ] [[package]] +name = "portable-atomic" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] name = "powerfmt" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2362,6 +2490,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "unit-prefix" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "323402cff2dd658f39ca17c789b502021b3f18707c91cdf22e3838e1b4023817" + +[[package]] name = "utf8parse" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2409,6 +2549,20 @@ dependencies = [ ] [[package]] +name = "warp_headless" +version = "0.1.0" +dependencies = [ + "binaryninja", + "binaryninjacore-sys", + "clap", + "ctrlc", + "env_logger", + "indicatif", + "log", + "warp_ninja", +] + +[[package]] name = "warp_ninja" version = "0.1.0" dependencies = [ @@ -2552,6 +2706,16 @@ dependencies = [ ] [[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] name = "weezl" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2589,6 +2753,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] name = "windows-sys" version = "0.59.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -2607,6 +2777,15 @@ dependencies = [ ] [[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] name = "windows-targets" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -22,6 +22,7 @@ members = [ "plugins/svd/demo", "plugins/warp", "plugins/warp/demo", + "plugins/warp/examples/headless", "plugins/workflow_objc", "plugins/workflow_objc/demo", ] @@ -29,6 +30,7 @@ members = [ [workspace.dependencies] binaryninja = { path = "rust" } binaryninjacore-sys = { path = "rust/binaryninjacore-sys" } +warp_ninja = { path = "plugins/warp" } [profile.release] lto = "thin" diff --git a/plugins/warp/Cargo.toml b/plugins/warp/Cargo.toml index 1e4ecb5e..7a7df157 100644 --- a/plugins/warp/Cargo.toml +++ b/plugins/warp/Cargo.toml @@ -18,7 +18,7 @@ log = "0.4" itertools = "0.14.0" dashmap = { version = "6.1", features = ["rayon"]} rayon = "1.10" -arboard = "3.4" +arboard = {version = "3.4", default-features = false} walkdir = "2.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/plugins/warp/README.md b/plugins/warp/README.md index 6506a7c3..d1193a9e 100644 --- a/plugins/warp/README.md +++ b/plugins/warp/README.md @@ -2,43 +2,6 @@ Provides integration with [WARP](https://github.com/Vector35/warp), more specifically the function identification and associated type information. -## Batch Processing +## Documentation -`sigem` (see code at `src/bin/sigem.rs`) allows for batch processing binaries for signature generation. - -The only real "magic" here is the cross-linking of symbols in archives, this means that static libraries will have much finer constraint matching. - -Duplicate functions will be removed to save space, any input will always produce a single output signature file, if you want separate signature files, invoke individually. - -### Building - -> Assuming you have the following: -> - A compatible Binary Ninja with headless usage (see [this documentation](https://docs.binary.ninja/dev/batch.html#batch-processing-and-other-automation-tips) for more information) -> - Clang -> - Rust (currently tested for 1.83.0) -> - Set `BINARYNINJADIR` env variable to your install directory (see [here](https://docs.binary.ninja/guide/#binary-path) for more details) - -1. Clone this repository (`git clone https://github.com/Vector35/binaryninja-api/tree/dev`) -2. Navigate to this plugin (`cd plugins/warp`) -3. Build in release (`cargo build --release`) - -If compilation fails because it could not link against binaryninjacore than you should double-check you set `BINARYNINJADIR` correctly. - -Once it finishes you now will have a `sigem` binary in `target/release` for use. - -### Usage - -> Assuming you already have the `sigem` binary and a valid headless compatible Binary Ninja license. - -To create a signature file simply pass the input as the first positional argument to `sigem`: - -- A regular binary -- An archive of binaries (`.a`, `.lib`, `.rlib`) -- A directory of binaries -- A BNDB - -Example: `./sigem mylibrary.a` or `./sigem ./all-libs/` - -Once its finished you should see a `.sbin` file next to the input file, this can be moved into the corresponding signature folder (see the [user docs](https://docs.binary.ninja/dev/annotation.html?h=install+path#signature-library) for more info) - -If you encounter malloc errors or instability try and adjust the number of parallel threads using `RAYON_NUM_THREADS` environment variable (ex. `RAYON_NUM_THREADS=1 ./sigem mylib.a`)
\ No newline at end of file +Documentation is available in the [User Guide](../../docs/guide/warp.md).
\ No newline at end of file diff --git a/plugins/warp/examples/headless/Cargo.toml b/plugins/warp/examples/headless/Cargo.toml new file mode 100644 index 00000000..b28b089d --- /dev/null +++ b/plugins/warp/examples/headless/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "warp_headless" +version = "0.1.0" +edition = "2021" + +[dependencies] +clap = { version = "4.5", features = ["derive"] } +indicatif = "0.18.2" +warp_ninja.workspace = true +binaryninjacore-sys.workspace = true +binaryninja.workspace = true +log = "0.4" +env_logger = "0.11" +ctrlc = "3.5"
\ No newline at end of file diff --git a/plugins/warp/examples/headless/README.md b/plugins/warp/examples/headless/README.md new file mode 100644 index 00000000..be4278b5 --- /dev/null +++ b/plugins/warp/examples/headless/README.md @@ -0,0 +1,44 @@ +# Headless WARP Processor + +Provides headless support for generating WARP signatures using Binary Ninja. + +### Building + +> Assuming you have the following: +> - A compatible Binary Ninja with headless usage (see [this documentation](https://docs.binary.ninja/dev/batch.html#batch-processing-and-other-automation-tips) for more information) +> - Clang +> - Rust (currently tested for 1.83.0) +> - Set `BINARYNINJADIR` env variable to your installation directory (see [here](https://docs.binary.ninja/guide/#binary-path) for more details) +> - If this is not set, the -sys crate will try and locate using the default installation path and last run location. + +1. Clone this repository (`git clone https://github.com/Vector35/binaryninja-api/tree/dev`) +2. Build in release (`cargo build --release`) + +If compilation fails because it could not link against binaryninjacore than you should double-check you set `BINARYNINJADIR` correctly. + +Once it finishes you now will have a `warp_headless` binary in `target/release` for use. + +### Usage + +> Assuming you already have the `warp_headless` binary and a valid headless compatible Binary Ninja license. + +To create a signature file simply pass the input as the first positional argument to `warp_headless`, as well as the output file using `-o` or `--output`: + +- A regular binary +- An archive of binaries (`.a`, `.lib`, `.rlib`) +- A directory of binaries +- A BNDB +- A WARP file + +Examples: + +- `./warp_headless -o ./results mylibrary.a` + - Places a `mylibrary.warp` file in the `results` directory +- `./warp_headless -o ./signatures.warp ./all-libs/` + - Provided files can be merged into a single output file +- `./warp_headless -o ./signatures.warp ./all-libs/ mylibrary.bndb` + - Multiple inputs can be provided in any order + +Once its finished you should see a `.warp` file in the provided output location, this can be moved into the corresponding signature folder (see the [user docs](https://docs.binary.ninja/dev/annotation.html?h=install+path#signature-library) for more info) + +If you encounter malloc errors or instability try and adjust the number of parallel threads using `RAYON_NUM_THREADS` environment variable (ex. `RAYON_NUM_THREADS=1 ./warp_headless -o ./sigs.warp mylib.a`)
\ No newline at end of file diff --git a/plugins/warp/examples/headless/src/main.rs b/plugins/warp/examples/headless/src/main.rs new file mode 100644 index 00000000..c08103a5 --- /dev/null +++ b/plugins/warp/examples/headless/src/main.rs @@ -0,0 +1,182 @@ +use binaryninja::headless::Session; +use clap::Parser; +use indicatif::{ProgressBar, ProgressStyle}; +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use warp_ninja::processor::{ + CompressionTypeField, ProcessingFileState, ProcessingState, WarpFileProcessor, +}; +use warp_ninja::warp::WarpFile; + +/// Generate WARP files using Binary Ninja +/// +/// Examples: +/// +/// - Merge multiple inputs into a single output: +/// ./binaryninja-warp-headless -o result.warp ./bin1 ./bin2 +/// +/// - Write one .warp per input into a directory: +/// ./binaryninja-warp-headless -o ./results ./bin1 ./bin2 ./dir_with_bins +/// +/// - Skip already-existing .warp files inside inputs: +/// ./binaryninja-warp-headless -o ./results --skip-warp-files ./dir_with_mixed_inputs +/// +/// - Use a cache directory to speed up repeated runs: +/// ./binaryninja-warp-headless -o ./results --cache-dir ./bn-cache ./binaries +#[derive(Parser, Debug)] +#[command(version, about, long_about)] +struct Args { + /// Input files and directories (positional). + #[arg(value_name = "PATH", required = true)] + input: Vec<PathBuf>, + + /// Output destination: file path or directory for per-input output + #[arg(short = 'o', long = "output")] + output: PathBuf, + + /// Skip processing of any existing WARP files within any input path + #[arg(long, default_value_t = false)] + skip_warp_files: bool, + + /// Optional directory to cache analysis information to + #[arg(long)] + cache_dir: Option<PathBuf>, + + /// Whether to compress the output file data + #[arg(short, long, default_value_t = true)] + compressed: bool, +} + +fn main() { + let _session = Session::new().expect("Failed to create session"); + let args = Args::parse(); + env_logger::init(); + + let compression_ty = match args.compressed { + true => CompressionTypeField::Zstd, + false => CompressionTypeField::None, + }; + let mut processor = WarpFileProcessor::new() + .with_skip_warp_files(args.skip_warp_files) + .with_compression_type(compression_ty); + if let Some(cache_dir) = args.cache_dir { + processor = processor.with_cache_path(cache_dir); + } + + // Cancel the processor on ctrl+c, otherwise it will block forever. + let ctrlc_state = processor.state().clone(); + ctrlc::set_handler(move || { + ctrlc_state.cancel(); + }) + .expect("Error setting Ctrl-C handler"); + + let progress_guard = run_progress_bar(processor.state()); + + let outputs: HashMap<PathBuf, WarpFile<'static>> = args + .input + .into_iter() + .filter_map(|i| match processor.process(i.clone()) { + Ok(o) => Some((i, o)), + Err(err) => { + log::error!("{}", err); + None + } + }) + .collect(); + + // Stop progress UI + progress_guard.finish(); + + match args.output.is_dir() { + true => { + // Given a directory, place each output individually using the input name. + for (input, output) in &outputs { + let output_name = match input.file_name() { + Some(name) => name, + None => input.components().last().unwrap().as_os_str(), + }; + let output_path = args.output.join(output_name).with_extension("warp"); + log::info!("Writing to {:?}", output_path); + std::fs::write(output_path, output.to_bytes()).unwrap(); + } + } + false => { + // Given a non-existing directory, merge all outputs and place at the output path. + match processor.merge_files(outputs.values().cloned().collect()) { + Ok(output) => { + log::info!("Writing to {:?}", args.output); + std::fs::write(args.output, output.to_bytes()).unwrap(); + } + Err(err) => { + log::error!("{}", err); + } + } + } + } +} + +// TODO: Also poll for background tasks and display them as independent progress bars. +fn run_progress_bar(state: Arc<ProcessingState>) -> ProgressGuard { + let pb = ProgressBar::new(0); + pb.set_style( + ProgressStyle::with_template( + "{spinner} {bar:40.cyan/blue} {pos}/{len} ({percent}%) [{elapsed_precise}] {msg}", + ) + .unwrap() + .progress_chars("=>-"), + ); + + let pb_clone = pb.clone(); + let state_clone = state.clone(); + let handle = std::thread::spawn(move || loop { + let total = state_clone.total_files() as u64; + let done = state_clone.files_with_state(ProcessingFileState::Processed) as u64; + let unprocessed = state_clone.files_with_state(ProcessingFileState::Unprocessed); + let analyzing = state_clone.files_with_state(ProcessingFileState::Analyzing); + let processing = state_clone.files_with_state(ProcessingFileState::Processing); + + if pb_clone.length().unwrap_or(0) != total { + pb_clone.set_length(total); + } + pb_clone.set_position(done.min(total)); + pb_clone.set_message(format!( + "{{u:{}|a:{}|p:{}|d:{}}}", + unprocessed, analyzing, processing, done + )); + + if pb_clone.is_finished() { + break; + } + std::thread::sleep(Duration::from_millis(100)); + }); + + ProgressGuard { + pb, + handle: Some(handle), + } +} + +struct ProgressGuard { + pb: ProgressBar, + handle: Option<std::thread::JoinHandle<()>>, +} + +impl ProgressGuard { + fn finish(mut self) { + self.pb.finish_and_clear(); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} + +impl Drop for ProgressGuard { + fn drop(&mut self) { + self.pb.finish_and_clear(); + if let Some(h) = self.handle.take() { + let _ = h.join(); + } + } +} |