diff options
| author | Mason Reed <mason@vector35.com> | 2026-02-11 18:04:07 -0800 |
|---|---|---|
| committer | Mason Reed <35282038+emesare@users.noreply.github.com> | 2026-02-23 00:09:44 -0800 |
| commit | 37008b7fa16837d04c1658868646cad681cbe035 (patch) | |
| tree | 577c2b62ee47c78a5d31d11f2aa610e441f5c808 /plugins/bntl_utils | |
| parent | 837f8590be80b7c98162e70e4f0c1814b83e9d7b (diff) | |
Add BNTL utility plugin
Allow users to easily create, diff, dump and validate type libraries
Supports the following formats:
- C header files (via core type parsers)
- Binary files (collects exported and imported functions)
- WinMD files (via `windows-metadata` crate)
- Existing type library files (for easy fixups)
- Apiset files (to resolve through forwarded windows dlls)
Can be invoked as a regular plugin via UI commands or via CLI.
Processing of type libraries inherently requires external linking, processing will automatically merge and deduplicate colliding type libraries so prefer to use inside a project or a directory and process all information (for a given platform) at once, rather than smaller invocations.
Diffstat (limited to 'plugins/bntl_utils')
32 files changed, 6300 insertions, 0 deletions
diff --git a/plugins/bntl_utils/CMakeLists.txt b/plugins/bntl_utils/CMakeLists.txt new file mode 100644 index 00000000..b1573e37 --- /dev/null +++ b/plugins/bntl_utils/CMakeLists.txt @@ -0,0 +1,168 @@ +cmake_minimum_required(VERSION 3.15 FATAL_ERROR) + +project(bntl_utils) + +if(NOT BN_API_BUILD_EXAMPLES AND NOT BN_INTERNAL_BUILD) + if(NOT BN_API_PATH) + # If we have not already defined the API source directory try and find it. + find_path( + BN_API_PATH + NAMES binaryninjaapi.h + # List of paths to search for the clone of the api + HINTS ../../.. ../../binaryninja/api/ binaryninjaapi binaryninja-api $ENV{BN_API_PATH} + REQUIRED + ) + endif() + set(CARGO_STABLE_VERSION 1.91.1) + add_subdirectory(${BN_API_PATH} binaryninjaapi) +endif() + +file(GLOB_RECURSE PLUGIN_SOURCES CONFIGURE_DEPENDS + ${PROJECT_SOURCE_DIR}/Cargo.toml + ${PROJECT_SOURCE_DIR}/src/*.rs) + +if(CMAKE_BUILD_TYPE MATCHES Debug) + if(DEMO) + set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/dev-demo) + set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target --profile=dev-demo) + else() + set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/debug) + set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target) + endif() +else() + if(DEMO) + set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/release-demo) + set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target --profile=release-demo) + else() + set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/release) + set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target --release) + endif() +endif() + +if(FORCE_COLORED_OUTPUT) + set(CARGO_OPTS ${CARGO_OPTS} --color always) +endif() + +if(DEMO) + set(CARGO_FEATURES --features demo --manifest-path ${PROJECT_SOURCE_DIR}/demo/Cargo.toml) + + set(OUTPUT_FILE_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}_static${CMAKE_STATIC_LIBRARY_SUFFIX}) + set(OUTPUT_PDB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}.pdb) + set(OUTPUT_FILE_PATH ${CMAKE_BINARY_DIR}/${OUTPUT_FILE_NAME}) + set(OUTPUT_PDB_PATH ${CMAKE_BINARY_DIR}/${OUTPUT_PDB_NAME}) + + set(BINJA_LIB_DIR $<TARGET_FILE_DIR:binaryninjacore>) +else() + # NOTE: --no-default-features is set to disable building artifacts used for testing + # NOTE: the linker is looking in the target dir and linking on it apparently. + set(CARGO_FEATURES "--no-default-features") + + set(OUTPUT_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${PROJECT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX}) + set(OUTPUT_PDB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${PROJECT_NAME}.pdb) + set(OUTPUT_FILE_PATH ${BN_CORE_PLUGIN_DIR}/${OUTPUT_FILE_NAME}) + set(OUTPUT_PDB_PATH ${BN_CORE_PLUGIN_DIR}/${OUTPUT_PDB_NAME}) + + set(BINJA_LIB_DIR ${BN_INSTALL_BIN_DIR}) +endif() + + +add_custom_target(${PROJECT_NAME} ALL DEPENDS ${OUTPUT_FILE_PATH}) +add_dependencies(${PROJECT_NAME} binaryninjaapi) +get_target_property(BN_API_SOURCE_DIR binaryninjaapi SOURCE_DIR) +list(APPEND CMAKE_MODULE_PATH "${BN_API_SOURCE_DIR}/cmake") +find_package(BinaryNinjaCore REQUIRED) + +set_property(TARGET ${PROJECT_NAME} PROPERTY OUTPUT_FILE_PATH ${OUTPUT_FILE_PATH}) + +# Add the whole api to the depends too +file(GLOB API_SOURCES CONFIGURE_DEPENDS + ${BN_API_SOURCE_DIR}/binaryninjacore.h + ${BN_API_SOURCE_DIR}/rust/src/*.rs + ${BN_API_SOURCE_DIR}/rust/binaryninjacore-sys/src/*.rs) + +find_program(RUSTUP_PATH rustup REQUIRED HINTS ~/.cargo/bin) +set(RUSTUP_COMMAND ${RUSTUP_PATH} run ${CARGO_STABLE_VERSION} cargo) + +if(APPLE) + if(UNIVERSAL) + if(CMAKE_BUILD_TYPE MATCHES Debug) + if(DEMO) + set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/dev-demo/${OUTPUT_FILE_NAME}) + set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/dev-demo/${OUTPUT_FILE_NAME}) + else() + set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/debug/${OUTPUT_FILE_NAME}) + set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/debug/${OUTPUT_FILE_NAME}) + endif() + else() + if(DEMO) + set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/release-demo/${OUTPUT_FILE_NAME}) + set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/release-demo/${OUTPUT_FILE_NAME}) + else() + set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/release/${OUTPUT_FILE_NAME}) + set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/release/${OUTPUT_FILE_NAME}) + endif() + endif() + + add_custom_command( + OUTPUT ${OUTPUT_FILE_PATH} + COMMAND ${CMAKE_COMMAND} -E env + MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR} + ${RUSTUP_COMMAND} clean --target=aarch64-apple-darwin ${CARGO_OPTS} --package binaryninjacore-sys + COMMAND ${CMAKE_COMMAND} -E env + MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR} + ${RUSTUP_COMMAND} clean --target=x86_64-apple-darwin ${CARGO_OPTS} --package binaryninjacore-sys + COMMAND ${CMAKE_COMMAND} -E env + MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR} + ${RUSTUP_COMMAND} build --target=aarch64-apple-darwin ${CARGO_OPTS} ${CARGO_FEATURES} + COMMAND ${CMAKE_COMMAND} -E env + MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR} + ${RUSTUP_COMMAND} build --target=x86_64-apple-darwin ${CARGO_OPTS} ${CARGO_FEATURES} + COMMAND lipo -create ${AARCH64_LIB_PATH} ${X86_64_LIB_PATH} -output ${OUTPUT_FILE_PATH} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES} + ) + else() + add_custom_command( + OUTPUT ${OUTPUT_FILE_PATH} + COMMAND ${CMAKE_COMMAND} -E env + MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR} + ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys + COMMAND ${CMAKE_COMMAND} -E env + MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR} + ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES} + COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES} + ) + endif() +elseif(WIN32) + if(DEMO) + add_custom_command( + OUTPUT ${OUTPUT_FILE_PATH} + COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys + COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES} + COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES} + ) + else() + add_custom_command( + OUTPUT ${OUTPUT_FILE_PATH} + COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys + COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES} + COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH} + COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_PDB_NAME} ${OUTPUT_PDB_PATH} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES} + ) + endif() +else() + add_custom_command( + OUTPUT ${OUTPUT_FILE_PATH} + COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys + COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES} + COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH} + WORKING_DIRECTORY ${PROJECT_SOURCE_DIR} + DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES} + ) +endif() diff --git a/plugins/bntl_utils/Cargo.toml b/plugins/bntl_utils/Cargo.toml new file mode 100644 index 00000000..0f867847 --- /dev/null +++ b/plugins/bntl_utils/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "bntl_utils" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +publish = false + +[lib] +crate-type = ["cdylib", "lib"] + +[dependencies] +binaryninja.workspace = true +binaryninjacore-sys.workspace = true +tracing = "0.1" +thiserror = "2.0" +similar = "2.7.0" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +nt-apiset = "0.1.0" +url = "2.5" +uuid = "1.20" +walkdir = "2.5" +dashmap = "6.1" + +# For TBD parsing +serde-saphyr = { version = "0.0.18", default-features = false, features = [] } + +# For reports +minijinja = "2.10.2" +minijinja-embed = "2.10.2" + +[build-dependencies] +minijinja-embed = "2.10.2" + +# TODO: We need to depend on latest because the windows-metadata crate has not yet been bumped, but depending on the crate +# TODO: with git will mean we pull in all of the data of the crate instead of just the necessary bits, we likely need to +# TODO: wait until the windows-metadata crate is bumped before merging this PR. +# TODO: Relevant PR: https://github.com/microsoft/windows-rs/pull/3799 +# TODO: Relevant issue: https://github.com/microsoft/windows-rs/issues/3887 +[dependencies.windows-metadata] +git = "https://github.com/microsoft/windows-rs" +tag = "72"
\ No newline at end of file diff --git a/plugins/bntl_utils/README.md b/plugins/bntl_utils/README.md new file mode 100644 index 00000000..2ad2ab4b --- /dev/null +++ b/plugins/bntl_utils/README.md @@ -0,0 +1,5 @@ +# BNTL Utilities + +A plugin and CLI tool for processing Binary Ninja type libraries (BNTL). + +For CLI build instructions and usage see [here](./cli/README.md).
\ No newline at end of file diff --git a/plugins/bntl_utils/build.rs b/plugins/bntl_utils/build.rs new file mode 100644 index 00000000..9165a33e --- /dev/null +++ b/plugins/bntl_utils/build.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; + +fn main() { + let link_path = std::env::var_os("DEP_BINARYNINJACORE_PATH") + .expect("DEP_BINARYNINJACORE_PATH not specified"); + + println!("cargo::rustc-link-lib=dylib=binaryninjacore"); + println!("cargo::rustc-link-search={}", link_path.to_str().unwrap()); + + #[cfg(not(target_os = "windows"))] + { + println!( + "cargo::rustc-link-arg=-Wl,-rpath,{0},-L{0}", + link_path.to_string_lossy() + ); + } + + // #[cfg(target_os = "macos")] + // { + // let crate_name = std::env::var("CARGO_PKG_NAME").expect("CARGO_PKG_NAME not set"); + // let lib_name = crate_name.replace('-', "_"); + // println!( + // "cargo::rustc-link-arg=-Wl,-install_name,@rpath/lib{}.dylib", + // lib_name + // ); + // } + + let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR specified"); + let out_dir_path = PathBuf::from(out_dir); + + // Copy all binaries to OUT_DIR for unit tests. + let bin_dir: PathBuf = "fixtures/".into(); + if let Ok(entries) = std::fs::read_dir(bin_dir) { + for entry in entries { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_file() { + let file_name = path.file_name().unwrap(); + let dest_path = out_dir_path.join(file_name); + std::fs::copy(&path, &dest_path).expect("failed to copy binary to OUT_DIR"); + } + } + } + + println!("cargo::rerun-if-changed=src/templates"); + // Templates used for rendering reports. + minijinja_embed::embed_templates!("src/templates"); +} diff --git a/plugins/bntl_utils/cli/Cargo.toml b/plugins/bntl_utils/cli/Cargo.toml new file mode 100644 index 00000000..863b950d --- /dev/null +++ b/plugins/bntl_utils/cli/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "bntl_cli" +version = "0.1.0" +edition = "2024" + +[dependencies] +binaryninja.workspace = true +binaryninjacore-sys.workspace = true +bntl_utils = { path = "../" } +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +clap = { version = "4.5.58", features = ["derive"] } +rayon = "1.11" +serde_json = "1.0" +thiserror = "2.0"
\ No newline at end of file diff --git a/plugins/bntl_utils/cli/README.md b/plugins/bntl_utils/cli/README.md new file mode 100644 index 00000000..5c4ddcb7 --- /dev/null +++ b/plugins/bntl_utils/cli/README.md @@ -0,0 +1,61 @@ +# Headless BNTL Processor + +Provides headless support for generating, inspecting, and validating Binary Ninja type libraries (BNTL). + +### Building + +> Assuming you have the following: +> - A compatible Binary Ninja with headless usage (see [this documentation](https://docs.binary.ninja/dev/batch.html#batch-processing-and-other-automation-tips) for more information) +> - Clang +> - Rust (currently tested for 1.91.1) +> - Set `BINARYNINJADIR` env variable to your installation directory (see [here](https://docs.binary.ninja/guide/#binary-path) for more details) + > - If this is not set, the -sys crate will try and locate using the default installation path and last run location. + +1. Clone this repository (`git clone https://github.com/Vector35/binaryninja-api/tree/dev`) +2. Build in release (`cargo build --release`) + +If compilation fails because it could not link against binaryninjacore than you should double-check you set `BINARYNINJADIR` correctly. + +Once it finishes you now will have a `bntl_cli` binary in `target/release` for use. + +### Usage + +> Assuming you already have the `bntl_cli` binary and a valid headless compatible Binary Ninja license. + +#### Create + +Generate a new type library from local files or remote projects. + +Examples: + +- `./bntl_cli create sqlite3.dll "windows-x86_64" ./headers/ ./output/` + - Places a single `sqlite.dll.bntl` file in the `output` directory, as headers have no dependency names associated they will be named `sqlite.dll`. +- `./bntl_cli create myproject "windows-x86_64" binaryninja://enterprise/https://enterprise.com/23ce5eaa-f532-4a93-80f2-a7d7f0aed040/ ./output/` + - Downloads and processes all files in the project, placing potentially multiple `.bntl` files in the `output` directory. +- `./bntl_cli create sqlite3.dll "windows-x86_64" ./winmd/ ./output/` + - `winmd` files are also supported as input, they will be processed together. You also probably want to provide some apiset schema files as well. + +#### Dump + +Export a type library back into a C header file for inspection. + +Examples: + +- `./bntl_cli dump sqlite3.dll.bntl ./output/sqlite.h` + +#### Diff + +Compare two type libraries and generate a .diff file containing a similarity ratio. + +Examples: + +- `./bntl_cli diff sqlite3.dll.bntl sqlite3.dll.bntl ./output/sqlite.diff` + +#### Validate + +Check type libraries for common errors, ensuring all referenced types exist across specified platforms. + +Examples: + +- `./bntl_cli validate ./typelibs/ ./output/` + - Pass in a directory containing `.bntl` files to validate, outputting a JSON file for each type library containing any errors. diff --git a/plugins/bntl_utils/cli/build.rs b/plugins/bntl_utils/cli/build.rs new file mode 100644 index 00000000..ed6cec7d --- /dev/null +++ b/plugins/bntl_utils/cli/build.rs @@ -0,0 +1,15 @@ +fn main() { + let link_path = std::env::var_os("DEP_BINARYNINJACORE_PATH") + .expect("DEP_BINARYNINJACORE_PATH not specified"); + + println!("cargo::rustc-link-lib=dylib=binaryninjacore"); + println!("cargo::rustc-link-search={}", link_path.to_str().unwrap()); + + #[cfg(not(target_os = "windows"))] + { + println!( + "cargo::rustc-link-arg=-Wl,-rpath,{0},-L{0}", + link_path.to_string_lossy() + ); + } +} diff --git a/plugins/bntl_utils/cli/src/create.rs b/plugins/bntl_utils/cli/src/create.rs new file mode 100644 index 00000000..a56b1cdb --- /dev/null +++ b/plugins/bntl_utils/cli/src/create.rs @@ -0,0 +1,79 @@ +use crate::input::{Input, ResolvedInput}; +use binaryninja::platform::Platform; +use bntl_utils::process::TypeLibProcessor; +use clap::Args; +use std::path::PathBuf; + +#[derive(Debug, Args)] +pub struct CreateArgs { + /// The name of the type library to create. + /// + /// TODO: Note that this wont be used for inputs which provide a name + pub name: String, + /// TODO: Note that this wont be used for inputs which provide a platform + pub platform: String, + pub input: Input, + pub output_directory: Option<PathBuf>, + #[clap(long)] + pub dry_run: bool, +} + +impl CreateArgs { + pub fn execute(&self) { + let Some(_platform) = Platform::by_name(&self.platform) else { + tracing::error!("Failed to find platform: {}", self.platform); + let platforms: Vec<_> = Platform::list_all().iter().map(|p| p.name()).collect(); + tracing::error!("Available platforms: {}", platforms.join(", ")); + panic!("Platform not found"); + }; + + let output_path = self + .output_directory + .clone() + .unwrap_or(PathBuf::from("./output/")); + if output_path.exists() && !output_path.is_dir() { + tracing::error!("Output path {} is not a directory", output_path.display()); + return; + } + std::fs::create_dir_all(&output_path).expect("Failed to create output directory"); + + let processor = TypeLibProcessor::new(&self.name, &self.platform); + // TODO: Need progress indicator here, when downloading files. + let resolved_input = self.input.resolve().expect("Failed to resolve input"); + + let data = match resolved_input { + ResolvedInput::Path(path) => processor.process(&path), + ResolvedInput::Project(project) => processor.process_project(&project), + ResolvedInput::ProjectFolder(project_folder) => { + processor.process_project_folder(&project_folder) + } + ResolvedInput::ProjectFile(project_file) => { + processor.process_project_file(&project_file) + } + } + .expect("Failed to process input"); + + if self.dry_run { + tracing::info!("Dry run enabled, skipping actual type library creation"); + return; + } + + for type_library in data.type_libraries { + // Place the type libraries in a folder with the architecture name, as that is necessary + // information for the user to correctly place the following type libraries in the user directory. + let arch_output_path = output_path.join(type_library.arch().name()); + std::fs::create_dir_all(&arch_output_path) + .expect("Failed to create architecture directory"); + let output_path = arch_output_path.join(format!("{}.bntl", type_library.name())); + if type_library.write_to_file(&output_path) { + tracing::info!( + "Created type library '{}': {}", + type_library.name(), + output_path.display() + ); + } else { + tracing::error!("Failed to write type library to {}", output_path.display()); + } + } + } +} diff --git a/plugins/bntl_utils/cli/src/diff.rs b/plugins/bntl_utils/cli/src/diff.rs new file mode 100644 index 00000000..1eedb970 --- /dev/null +++ b/plugins/bntl_utils/cli/src/diff.rs @@ -0,0 +1,37 @@ +use binaryninja::types::TypeLibrary; +use bntl_utils::diff::TILDiff; +use clap::Args; +use std::path::PathBuf; + +#[derive(Debug, Args)] +pub struct DiffArgs { + pub file_a: PathBuf, + pub file_b: PathBuf, + /// Path to write the `.diff` file to. + pub output_path: PathBuf, + /// Timeout in seconds for the diff operation to complete, if provided the diffing will begin + /// to approximate after the deadline has passed. + #[clap(long)] + pub timeout: Option<u64>, +} + +impl DiffArgs { + pub fn execute(&self) { + let type_lib_a = + TypeLibrary::load_from_file(&self.file_a).expect("Failed to load type library"); + let type_lib_b = + TypeLibrary::load_from_file(&self.file_b).expect("Failed to load type library"); + + let diff_result = + match TILDiff::new().diff((&self.file_a, &type_lib_a), (&self.file_b, &type_lib_b)) { + Ok(diff_result) => diff_result, + Err(err) => { + tracing::error!("Failed to diff type libraries: {}", err); + return; + } + }; + tracing::info!("Similarity Ratio: {}", diff_result.ratio); + std::fs::write(&self.output_path, diff_result.diff).unwrap(); + tracing::info!("Diff written to: {}", self.output_path.display()); + } +} diff --git a/plugins/bntl_utils/cli/src/dump.rs b/plugins/bntl_utils/cli/src/dump.rs new file mode 100644 index 00000000..fb25446b --- /dev/null +++ b/plugins/bntl_utils/cli/src/dump.rs @@ -0,0 +1,26 @@ +use binaryninja::types::TypeLibrary; +use bntl_utils::dump::TILDump; +use clap::Args; +use std::path::PathBuf; + +#[derive(Debug, Args)] +pub struct DumpArgs { + pub input: PathBuf, + pub output_path: Option<PathBuf>, +} + +impl DumpArgs { + pub fn execute(&self) { + let type_lib = + TypeLibrary::load_from_file(&self.input).expect("Failed to load type library"); + let default_output_path = self.input.with_extension("h"); + let output_path = self.output_path.as_ref().unwrap_or(&default_output_path); + let dependencies = + bntl_utils::helper::path_to_type_libraries(&self.input.parent().unwrap()); + let printed_types = TILDump::new() + .with_type_libs(dependencies) + .dump(&type_lib) + .expect("Failed to dump type library"); + std::fs::write(output_path, printed_types).expect("Failed to write type library header"); + } +} diff --git a/plugins/bntl_utils/cli/src/input.rs b/plugins/bntl_utils/cli/src/input.rs new file mode 100644 index 00000000..1e1dbb6e --- /dev/null +++ b/plugins/bntl_utils/cli/src/input.rs @@ -0,0 +1,167 @@ +use binaryninja::collaboration::RemoteFile; +use binaryninja::project::Project; +use binaryninja::project::file::ProjectFile; +use binaryninja::project::folder::ProjectFolder; +use binaryninja::rc::Ref; +use bntl_utils::url::{BnParsedUrl, BnResource}; +use std::fmt::Display; +use std::path::PathBuf; +use std::str::FromStr; +use thiserror::Error; + +#[derive(Debug)] +pub enum ResolvedInput { + Path(PathBuf), + Project(Ref<Project>), + ProjectFolder(Ref<ProjectFolder>), + ProjectFile(Ref<ProjectFile>), +} + +#[derive(Error, Debug)] +pub enum InputResolveError { + #[error("Resource resolution failed: {0}")] + ResourceError(#[from] bntl_utils::url::BnResourceError), + + #[error("Collaboration API error: {0}")] + CollaborationError(String), + + #[error("Download failed for {url}: status {status}")] + DownloadFailed { url: String, status: u16 }, + + #[error("Download provider error: {0}")] + DownloadProviderError(String), + + #[error("I/O error: {0}")] + Io(#[from] std::io::Error), + + #[error("Environment error: {0}")] + EnvError(String), +} + +/// An input to the CLI to locate a "resource", such as a file or directory. +#[derive(Debug, Clone)] +pub enum Input { + /// A URL which references a Binary Ninja resource, such as a remote project or file. + ParsedUrl(BnParsedUrl), + /// A local filesystem path pointing to a file or directory. + LocalPath(PathBuf), +} + +impl Input { + /// Attempt to acquire a path from this input, this can download files over the network and + /// is meant to be called when the file contents are desired. + pub fn resolve(&self) -> Result<ResolvedInput, InputResolveError> { + let try_download_file = |file: &RemoteFile| -> Result<(), InputResolveError> { + if !file.core_file().unwrap().exists_on_disk() { + let _span = + tracing::info_span!("Downloading project file", file = %file.name()).entered(); + file.download().map_err(|_| { + InputResolveError::CollaborationError("Failed to download project file".into()) + })?; + } + Ok(()) + }; + + match self { + Input::ParsedUrl(url) => match url.to_resource()? { + BnResource::RemoteProject(project) => { + let files = project.files().map_err(|_| { + InputResolveError::CollaborationError("Failed to get files".into()) + })?; + + for file in &files { + try_download_file(&file)?; + } + + let core = project.core_project().map_err(|_| { + InputResolveError::CollaborationError("Missing core project".into()) + })?; + Ok(ResolvedInput::Project(core)) + } + + BnResource::RemoteProjectFile(file) => { + try_download_file(&file)?; + let core = file.core_file().expect("Missing core file"); + Ok(ResolvedInput::ProjectFile(core)) + } + + BnResource::RemoteProjectFolder(folder) => { + let project = folder.project().map_err(|_| { + InputResolveError::CollaborationError("Failed to get project".into()) + })?; + let files = project.files().map_err(|_| { + InputResolveError::CollaborationError("Failed to get files".into()) + })?; + + for file in &files { + if let Some(file_folder) = file.folder().ok().flatten() { + if file_folder == folder { + try_download_file(&file)?; + } + } + } + + let core = folder.core_folder().map_err(|_| { + InputResolveError::CollaborationError("Missing core folder".into()) + })?; + Ok(ResolvedInput::ProjectFolder(core)) + } + + BnResource::RemoteFile(url) => { + let safe_name = url.to_string().replace(['/', ':', '?'], "_"); + let cached_file_path = std::env::temp_dir().join(safe_name); + if cached_file_path.exists() { + return Ok(ResolvedInput::Path(cached_file_path)); + } + + let download_provider = binaryninja::download::DownloadProvider::try_default() + .expect("Failed to get default download provider"); + let mut instance = download_provider + .create_instance() + .expect("Failed to create download provider instance"); + let _span = + tracing::info_span!("Downloading remote file", url = %url).entered(); + let response = instance + .get(&url.to_string(), Vec::new()) + .map_err(|e| InputResolveError::DownloadProviderError(e.to_string()))?; + if response.is_success() { + std::fs::write(&cached_file_path, response.data)?; + Ok(ResolvedInput::Path(cached_file_path)) + } else { + Err(InputResolveError::DownloadFailed { + url: url.to_string(), + status: response.status_code, + }) + } + } + + BnResource::LocalFile(path) => Ok(ResolvedInput::Path(path.clone())), + }, + Input::LocalPath(path) => Ok(ResolvedInput::Path(path.clone())), + } + } +} + +impl FromStr for Input { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + // Try to parse as a Binary Ninja URL + if s.starts_with("binaryninja:") { + let url = BnParsedUrl::parse(s).map_err(|e| format!("URL Parse Error: {}", e))?; + return Ok(Input::ParsedUrl(url)); + } + + let path = PathBuf::from(s); + Ok(Input::LocalPath(path)) + } +} + +impl Display for Input { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Input::ParsedUrl(url) => write!(f, "{}", url), + Input::LocalPath(path) => write!(f, "{}", path.display()), + } + } +} diff --git a/plugins/bntl_utils/cli/src/main.rs b/plugins/bntl_utils/cli/src/main.rs new file mode 100644 index 00000000..1466da1f --- /dev/null +++ b/plugins/bntl_utils/cli/src/main.rs @@ -0,0 +1,71 @@ +use clap::Parser; +use tracing::level_filters::LevelFilter; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; + +mod create; +mod diff; +mod dump; +mod input; +mod validate; + +/// Generate, inspect, and validate Binary Ninja type libraries (BNTL) +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +struct Cli { + #[clap(subcommand)] + command: Command, +} + +#[derive(Parser, Debug)] +pub enum Command { + /// Create a new type library from a set of files. + Create(create::CreateArgs), + /// Dump the type library to a C header file. + Dump(dump::DumpArgs), + /// Generate a diff between two type libraries. + Diff(diff::DiffArgs), + /// Validate the type libraries for common errors. + Validate(validate::ValidateArgs), +} + +impl Command { + pub fn execute(&self) { + match self { + Command::Create(args) => { + args.execute(); + } + Command::Dump(args) => { + args.execute(); + } + Command::Diff(args) => { + args.execute(); + } + Command::Validate(args) => { + args.execute(); + } + } + } +} + +fn main() { + let cli = Cli::parse(); + tracing_subscriber::registry() + .with(tracing_subscriber::fmt::layer()) + .with( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); + + // Capture logs from Binary Ninja + let _listener = binaryninja::tracing::TracingLogListener::new().register(); + + // Initialize Binary Ninja, requires a headless compatible license like commercial or ultimate. + let _session = binaryninja::headless::Session::new() + .expect("Failed to create headless binary ninja session"); + + cli.command.execute(); +} diff --git a/plugins/bntl_utils/cli/src/validate.rs b/plugins/bntl_utils/cli/src/validate.rs new file mode 100644 index 00000000..743ca927 --- /dev/null +++ b/plugins/bntl_utils/cli/src/validate.rs @@ -0,0 +1,66 @@ +use binaryninja::platform::Platform; +use bntl_utils::validate::{TypeLibValidater, ValidateIssue}; +use clap::Args; +use rayon::prelude::*; +use std::collections::HashMap; +use std::path::PathBuf; + +#[derive(Debug, Args)] +pub struct ValidateArgs { + /// Path to the directory containing the type libraries to validate. + /// + /// This must contain all the type libraries referencable. + pub input: PathBuf, + /// Dump validation results to the directory specified. + #[clap(short, long)] + pub output: Option<PathBuf>, +} + +impl ValidateArgs { + pub fn execute(&self) { + if let Some(output_dir) = &self.output { + std::fs::create_dir_all(output_dir).expect("Failed to create output directory"); + } + + // TODO: For now we just pass all the type libraries in the containing input directory. + let type_libs = bntl_utils::helper::path_to_type_libraries(&self.input); + type_libs.par_iter().for_each(|type_lib| { + // We run validation per platform. This is to make sure that if we depend on platform + // types that they exist in each one of the specified platforms, not just one of them. + let mut platform_mapped_issues: HashMap<ValidateIssue, Vec<String>> = HashMap::new(); + let available_platforms = type_lib.platform_names(); + + for platform in &available_platforms { + let platform = Platform::by_name(platform).expect("Failed to load platform"); + let mut ctx = TypeLibValidater::new() + .with_type_libraries(type_libs.clone()) + .with_platform(&platform); + let result = ctx.validate(&type_lib); + for issue in &result.issues { + platform_mapped_issues + .entry(issue.clone()) + .or_default() + .push(platform.name().to_string()); + } + + if let Some(output_dir) = &self.output + && !result.issues.is_empty() + { + let dump_path = output_dir + .join(type_lib.name()) + .with_extension(format!("{}.problems.json", platform.name())); + let result = serde_json::to_string_pretty(&result.issues) + .expect("Failed to serialize result"); + std::fs::write(dump_path, result).expect("Failed to write validation result"); + } + } + + for (issue, platforms) in platform_mapped_issues { + match (available_platforms.len(), platforms.len()) { + (1, _) => tracing::error!("{}", issue), + _ => tracing::error!("{}: {}", platforms.join(", "), issue), + } + } + }); + } +} diff --git a/plugins/bntl_utils/src/command.rs b/plugins/bntl_utils/src/command.rs new file mode 100644 index 00000000..39f25b99 --- /dev/null +++ b/plugins/bntl_utils/src/command.rs @@ -0,0 +1,66 @@ +use binaryninja::interaction::{Form, FormInputField}; +use binaryninja::user_directory; +use std::path::PathBuf; + +pub mod create; +pub mod diff; +pub mod dump; +pub mod validate; +// TODO: Load? + +pub struct InputFileField; + +impl InputFileField { + pub fn field() -> FormInputField { + FormInputField::OpenFileName { + prompt: "File Path".to_string(), + // TODO: This is called extension but is really a filter. + extension: None, + default: None, + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<PathBuf> { + let field = form.get_field_with_name("File Path")?; + let field_value = field.try_value_string()?; + Some(PathBuf::from(field_value)) + } +} + +pub struct OutputDirectoryField; + +impl OutputDirectoryField { + pub fn field() -> FormInputField { + let type_lib_dir = user_directory().join("typelib"); + FormInputField::DirectoryName { + prompt: "Output Directory".to_string(), + default: Some(type_lib_dir.to_string_lossy().to_string()), + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<PathBuf> { + let field = form.get_field_with_name("Output Directory")?; + let field_value = field.try_value_string()?; + Some(PathBuf::from(field_value)) + } +} + +pub struct InputDirectoryField; + +impl InputDirectoryField { + pub fn field() -> FormInputField { + FormInputField::DirectoryName { + prompt: "Input Directory".to_string(), + default: None, + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<PathBuf> { + let field = form.get_field_with_name("Input Directory")?; + let field_value = field.try_value_string()?; + Some(PathBuf::from(field_value)) + } +} diff --git a/plugins/bntl_utils/src/command/create.rs b/plugins/bntl_utils/src/command/create.rs new file mode 100644 index 00000000..41b5517a --- /dev/null +++ b/plugins/bntl_utils/src/command/create.rs @@ -0,0 +1,277 @@ +use crate::command::{InputDirectoryField, OutputDirectoryField}; +use crate::process::{new_processing_state_background_thread, TypeLibProcessor}; +use crate::validate::TypeLibValidater; +use binaryninja::background_task::BackgroundTask; +use binaryninja::binary_view::{BinaryView, BinaryViewExt}; +use binaryninja::command::{Command, ProjectCommand}; +use binaryninja::interaction::{Form, FormInputField, MessageBoxButtonSet, MessageBoxIcon}; +use binaryninja::platform::Platform; +use binaryninja::project::Project; +use binaryninja::types::TypeLibrary; +use std::thread; + +pub struct CreateFromCurrentView; + +impl Command for CreateFromCurrentView { + fn action(&self, view: &BinaryView) { + let mut form = Form::new("Create From View"); + // TODO: The choice to select what types to include + form.add_field(OutputDirectoryField::field()); + if !form.prompt() { + return; + } + let output_dir = OutputDirectoryField::from_form(&form).unwrap(); + let Some(default_platform) = view.default_platform() else { + tracing::error!("No default platform set for view"); + return; + }; + + let file_path = view.file().file_path(); + let file_name = file_path.file_name().unwrap_or_default().to_string_lossy(); + let processor = TypeLibProcessor::new(&file_name, &default_platform.name()); + let data = match processor.process_view(file_path, view) { + Ok(data) => data, + Err(err) => { + tracing::error!("Failed to process view: {}", err); + return; + } + } + .prune(); + + let attached_libraries = view + .type_libraries() + .iter() + .map(|t| t.to_owned()) + .chain(data.type_libraries.iter().map(|t| t.to_owned())) + .collect::<Vec<_>>(); + let mut validator = TypeLibValidater::new() + .with_platform(&default_platform) + .with_type_libraries(attached_libraries); + + for type_library in data.type_libraries { + let output_path = output_dir.join(format!("{}.bntl", type_library.name())); + + let validation_result = validator.validate(&type_library); + if !validation_result.issues.is_empty() { + tracing::error!( + "Found {} issues in type library '{}'", + validation_result.issues.len(), + type_library.name() + ); + match validation_result.render_report() { + Ok(rendered) => { + view.show_html_report(&type_library.name(), &rendered, ""); + if let Err(e) = std::fs::write(output_path.with_extension("html"), rendered) + { + tracing::error!( + "Failed to write validation report to {}: {}", + output_path.display(), + e + ); + } + } + Err(err) => tracing::error!("Failed to render validation report: {}", err), + } + } + + if type_library.write_to_file(&output_path) { + tracing::info!( + "Created type library '{}': {}", + type_library.name(), + output_path.display() + ); + } else { + tracing::error!("Failed to write type library to {}", output_path.display()); + } + } + } + + fn valid(&self, _view: &BinaryView) -> bool { + true + } +} + +pub struct NameField; + +impl NameField { + pub fn field() -> FormInputField { + FormInputField::TextLine { + prompt: "Dependency Name".to_string(), + default: Some("foo.dll".to_string()), + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<String> { + let field = form.get_field_with_name("Dependency Name")?; + field.try_value_string() + } +} + +pub struct PlatformField; + +impl PlatformField { + pub fn field() -> FormInputField { + FormInputField::TextLine { + prompt: "Platform Name".to_string(), + default: Some("windows-x86_64".to_string()), + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<String> { + let field = form.get_field_with_name("Platform Name")?; + field.try_value_string() + } +} + +pub struct CreateFromDirectory; + +impl CreateFromDirectory { + pub fn execute() { + let mut form = Form::new("Create From Directory"); + // TODO: The choice to select what types to include + form.add_field(InputDirectoryField::field()); + form.add_field(PlatformField::field()); + form.add_field(NameField::field()); + form.add_field(OutputDirectoryField::field()); + if !form.prompt() { + return; + } + let input_dir = InputDirectoryField::from_form(&form).unwrap(); + let platform_name = PlatformField::from_form(&form).unwrap(); + let default_name = NameField::from_form(&form).unwrap(); + let output_dir = OutputDirectoryField::from_form(&form).unwrap(); + + let Some(default_platform) = Platform::by_name(&platform_name) else { + tracing::error!("Invalid platform name: {}", platform_name); + return; + }; + + let processor = TypeLibProcessor::new(&default_name, &default_platform.name()); + + let background_task = BackgroundTask::new("Processing started...", true); + new_processing_state_background_thread(background_task.clone(), processor.state()); + let data = processor.process_directory(&input_dir); + background_task.finish(); + + let pruned_data = match data { + // Prune off empty type libraries, no need to save them. + Ok(data) => data.prune(), + Err(err) => { + binaryninja::interaction::show_message_box( + "Failed to process directory", + &err.to_string(), + MessageBoxButtonSet::OKButtonSet, + MessageBoxIcon::ErrorIcon, + ); + tracing::error!("Failed to process directory: {}", err); + return; + } + }; + + for type_library in pruned_data.type_libraries { + // Place the type libraries in a folder with the architecture name, as that is necessary + // information for the user to correctly place the following type libraries in the user directory. + let arch_output_path = output_dir.join(type_library.arch().name()); + let _ = std::fs::create_dir_all(&arch_output_path); + let output_path = arch_output_path.join(format!("{}.bntl", type_library.name())); + if type_library.write_to_file(&output_path) { + tracing::info!( + "Created type library '{}': {}", + type_library.name(), + output_path.display() + ); + } else { + tracing::error!("Failed to write type library to {}", output_path.display()); + } + } + } +} + +impl Command for CreateFromDirectory { + fn action(&self, _view: &BinaryView) { + thread::spawn(move || { + CreateFromDirectory::execute(); + }); + } + + fn valid(&self, _view: &BinaryView) -> bool { + true + } +} + +pub struct CreateFromProject; + +impl CreateFromProject { + pub fn execute(project: &Project) { + let mut form = Form::new("Create From Project"); + // TODO: The choice to select what types to include + form.add_field(PlatformField::field()); + form.add_field(NameField::field()); + form.add_field(OutputDirectoryField::field()); + if !form.prompt() { + return; + } + let platform_name = PlatformField::from_form(&form).unwrap(); + let default_name = NameField::from_form(&form).unwrap(); + let output_dir = OutputDirectoryField::from_form(&form).unwrap(); + + let Some(default_platform) = Platform::by_name(&platform_name) else { + tracing::error!("Invalid platform name: {}", platform_name); + return; + }; + + let processor = TypeLibProcessor::new(&default_name, &default_platform.name()); + + let background_task = BackgroundTask::new("Processing started...", true); + new_processing_state_background_thread(background_task.clone(), processor.state()); + let data = processor.process_project(&project); + background_task.finish(); + + let mut finalized_data = match data { + // Prune off empty type libraries, no need to save them. + Ok(data) => data.finalized(&default_name), + Err(err) => { + binaryninja::interaction::show_message_box( + "Failed to process project", + &err.to_string(), + MessageBoxButtonSet::OKButtonSet, + MessageBoxIcon::ErrorIcon, + ); + tracing::error!("Failed to process project: {}", err); + return; + } + }; + + for type_library in finalized_data.type_libraries { + // Place the type libraries in a folder with the architecture name, as that is necessary + // information for the user to correctly place the following type libraries in the user directory. + let arch_output_path = output_dir.join(type_library.arch().name()); + let _ = std::fs::create_dir_all(&arch_output_path); + let output_path = arch_output_path.join(format!("{}.bntl", type_library.name())); + if type_library.write_to_file(&output_path) { + tracing::info!( + "Created type library '{}': {}", + type_library.name(), + output_path.display() + ); + } else { + tracing::error!("Failed to write type library to {}", output_path.display()); + } + } + } +} + +impl ProjectCommand for CreateFromProject { + fn action(&self, project: &Project) { + let owned_project = project.to_owned(); + thread::spawn(move || { + CreateFromProject::execute(&owned_project); + }); + } + + fn valid(&self, _project: &Project) -> bool { + true + } +} diff --git a/plugins/bntl_utils/src/command/diff.rs b/plugins/bntl_utils/src/command/diff.rs new file mode 100644 index 00000000..ae93dafe --- /dev/null +++ b/plugins/bntl_utils/src/command/diff.rs @@ -0,0 +1,103 @@ +use crate::command::OutputDirectoryField; +use crate::diff::TILDiff; +use binaryninja::background_task::BackgroundTask; +use binaryninja::binary_view::BinaryView; +use binaryninja::command::Command; +use binaryninja::interaction::{Form, FormInputField}; +use binaryninja::types::TypeLibrary; +use std::path::PathBuf; +use std::thread; + +pub struct InputFileAField; + +impl InputFileAField { + pub fn field() -> FormInputField { + FormInputField::OpenFileName { + prompt: "Library A".to_string(), + // TODO: This is called extension but is really a filter. + extension: Some("*.bntl".to_string()), + default: None, + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<PathBuf> { + let field = form.get_field_with_name("Library A")?; + let field_value = field.try_value_string()?; + Some(PathBuf::from(field_value)) + } +} + +pub struct InputFileBField; + +impl InputFileBField { + pub fn field() -> FormInputField { + FormInputField::OpenFileName { + prompt: "Library B".to_string(), + // TODO: This is called extension but is really a filter. + extension: Some("*.bntl".to_string()), + default: None, + value: None, + } + } + + pub fn from_form(form: &Form) -> Option<PathBuf> { + let field = form.get_field_with_name("Library B")?; + let field_value = field.try_value_string()?; + Some(PathBuf::from(field_value)) + } +} + +pub struct Diff; + +impl Diff { + pub fn execute() { + let mut form = Form::new("Diff type libraries"); + form.add_field(InputFileAField::field()); + form.add_field(InputFileBField::field()); + form.add_field(OutputDirectoryField::field()); + if !form.prompt() { + return; + } + let a_path = InputFileAField::from_form(&form).unwrap(); + let b_path = InputFileBField::from_form(&form).unwrap(); + let output_dir = OutputDirectoryField::from_form(&form).unwrap(); + + let _bg_task = BackgroundTask::new("Diffing type libraries...", false).enter(); + let Some(type_lib_a) = TypeLibrary::load_from_file(&a_path) else { + tracing::error!("Failed to load type library: {}", a_path.display()); + return; + }; + let Some(type_lib_b) = TypeLibrary::load_from_file(&b_path) else { + tracing::error!("Failed to load type library: {}", b_path.display()); + return; + }; + + let diff_result = match TILDiff::new().diff((&a_path, &type_lib_a), (&b_path, &type_lib_b)) + { + Ok(diff_result) => diff_result, + Err(err) => { + tracing::error!("Failed to diff type libraries: {}", err); + return; + } + }; + tracing::info!("Similarity Ratio: {}", diff_result.ratio); + let output_path = output_dir + .join(type_lib_a.dependency_name()) + .with_extension("diff"); + std::fs::write(&output_path, diff_result.diff).unwrap(); + tracing::info!("Diff written to: {}", output_path.display()); + } +} + +impl Command for Diff { + fn action(&self, _view: &BinaryView) { + thread::spawn(move || { + Diff::execute(); + }); + } + + fn valid(&self, _view: &BinaryView) -> bool { + true + } +} diff --git a/plugins/bntl_utils/src/command/dump.rs b/plugins/bntl_utils/src/command/dump.rs new file mode 100644 index 00000000..4b68cd79 --- /dev/null +++ b/plugins/bntl_utils/src/command/dump.rs @@ -0,0 +1,52 @@ +use crate::command::{InputFileField, OutputDirectoryField}; +use crate::dump::TILDump; +use crate::helper::path_to_type_libraries; +use binaryninja::binary_view::BinaryView; +use binaryninja::command::Command; +use binaryninja::interaction::Form; +use binaryninja::types::TypeLibrary; + +pub struct Dump; + +impl Command for Dump { + // TODO: We need a command type that does not require a binary view. + fn action(&self, _view: &BinaryView) { + let mut form = Form::new("Dump to C Header"); + // TODO: The choice to select what to include? + form.add_field(InputFileField::field()); + form.add_field(OutputDirectoryField::field()); + if !form.prompt() { + return; + } + let output_dir = OutputDirectoryField::from_form(&form).unwrap(); + let input_path = InputFileField::from_form(&form).unwrap(); + + let type_lib = match TypeLibrary::load_from_file(&input_path) { + Some(type_lib) => type_lib, + None => { + tracing::error!("Failed to load type library from {}", input_path.display()); + return; + } + }; + + // TODO: Currently we collect input path dependencies from the platform and the parent directory. + let dependencies = path_to_type_libraries(input_path.parent().unwrap()); + let dump = match TILDump::new().with_type_libs(dependencies).dump(&type_lib) { + Ok(dump) => dump, + Err(err) => { + tracing::error!("Failed to dump type library: {}", err); + return; + } + }; + + let output_path = output_dir.join(format!("{}.h", type_lib.name())); + if let Err(e) = std::fs::write(&output_path, dump) { + tracing::error!("Failed to write dump to {}: {}", output_path.display(), e); + } + tracing::info!("Dump written to {}", output_path.display()); + } + + fn valid(&self, _view: &BinaryView) -> bool { + true + } +} diff --git a/plugins/bntl_utils/src/command/validate.rs b/plugins/bntl_utils/src/command/validate.rs new file mode 100644 index 00000000..8f0095ae --- /dev/null +++ b/plugins/bntl_utils/src/command/validate.rs @@ -0,0 +1,78 @@ +use crate::helper::path_to_type_libraries; +use crate::validate::TypeLibValidater; +use binaryninja::binary_view::{BinaryView, BinaryViewExt}; +use binaryninja::command::Command; +use binaryninja::interaction::get_open_filename_input; +use binaryninja::platform::Platform; +use binaryninja::types::TypeLibrary; + +pub struct Validate; + +impl Command for Validate { + fn action(&self, _view: &BinaryView) { + let Some(input_path) = + get_open_filename_input("Select a type library to validate", "*.bntl") + else { + return; + }; + + let type_lib = match TypeLibrary::load_from_file(&input_path) { + Some(type_lib) => type_lib, + None => { + tracing::error!("Failed to load type library from {}", input_path.display()); + return; + } + }; + + // Type libraries should always have at least one platform associated with them. + if type_lib.platform_names().is_empty() { + tracing::error!("Type library {} has no platforms!", input_path.display()); + return; + } + + // TODO: Currently we collect input path dependencies from the platform and the parent directory. + let dependencies = path_to_type_libraries(input_path.parent().unwrap()); + + let validator = TypeLibValidater::new().with_type_libraries(dependencies); + // Validate for every platform so that we can find issues in lesser used platforms. + for platform_name in &type_lib.platform_names() { + let Some(platform) = Platform::by_name(platform_name) else { + tracing::error!("Failed to find platform with name {}", platform_name); + continue; + }; + let results = validator + .clone() + .with_platform(&platform) + .validate(&type_lib); + if results.issues.is_empty() { + tracing::info!( + "No issues found for type library {} on platform {}", + type_lib.name(), + platform_name + ); + continue; + } + let rendered = match results.render_report() { + Ok(rendered) => rendered, + Err(err) => { + tracing::error!("Failed to render validation report: {}", err); + continue; + } + }; + let out_path = input_path.with_extension(format!("{}.html", platform_name)); + let out_name = format!("{} ({})", type_lib.name(), platform_name); + _view.show_html_report(&out_name, &rendered, ""); + if let Err(e) = std::fs::write(out_path, rendered) { + tracing::error!( + "Failed to write validation report to {}: {}", + input_path.display(), + e + ); + } + } + } + + fn valid(&self, _view: &BinaryView) -> bool { + true + } +} diff --git a/plugins/bntl_utils/src/diff.rs b/plugins/bntl_utils/src/diff.rs new file mode 100644 index 00000000..18d4dce8 --- /dev/null +++ b/plugins/bntl_utils/src/diff.rs @@ -0,0 +1,83 @@ +use crate::dump::TILDump; +use crate::helper::path_to_type_libraries; +use binaryninja::types::TypeLibrary; +use similar::{Algorithm, TextDiff}; +use std::path::{Path, PathBuf}; +use std::time::Duration; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum TILDiffError { + #[error("Could not determine parent directory for path: {0}")] + InvalidPath(PathBuf), + + #[error("Failed to dump type library: {0}")] + DumpError(String), +} + +pub struct DiffResult { + pub ratio: f32, + pub diff: String, +} + +pub struct TILDiff { + timeout: Duration, +} + +impl TILDiff { + pub fn new() -> Self { + Self { + timeout: Duration::from_secs(180), + } + } + + pub fn with_timeout(mut self, timeout: Duration) -> Self { + self.timeout = timeout; + self + } + + pub fn diff( + &self, + (a_path, a_type_lib): (&Path, &TypeLibrary), + (b_path, b_type_lib): (&Path, &TypeLibrary), + ) -> Result<DiffResult, TILDiffError> { + let a_parent = a_path + .parent() + .ok_or_else(|| TILDiffError::InvalidPath(a_path.to_path_buf()))?; + let b_parent = b_path + .parent() + .ok_or_else(|| TILDiffError::InvalidPath(b_path.to_path_buf()))?; + + let a_dependencies = path_to_type_libraries(a_parent); + let b_dependencies = path_to_type_libraries(b_parent); + + let dumped_a = TILDump::new() + .with_type_libs(a_dependencies) + .dump(a_type_lib) + .map_err(|e| TILDiffError::DumpError(e.to_string()))?; + + let dumped_b = TILDump::new() + .with_type_libs(b_dependencies) + .dump(b_type_lib) + .map_err(|e| TILDiffError::DumpError(e.to_string()))?; + + let diff = TextDiff::configure() + .algorithm(Algorithm::Patience) + .timeout(self.timeout) + .diff_lines(&dumped_a, &dumped_b); + + let diff_content = diff + .unified_diff() + .context_radius(3) + .header( + a_path.to_string_lossy().as_ref(), + b_path.to_string_lossy().as_ref(), + ) + .to_string(); + + Ok(DiffResult { + ratio: diff.ratio(), + diff: diff_content, + }) + } +} diff --git a/plugins/bntl_utils/src/dump.rs b/plugins/bntl_utils/src/dump.rs new file mode 100644 index 00000000..ddc9f602 --- /dev/null +++ b/plugins/bntl_utils/src/dump.rs @@ -0,0 +1,146 @@ +use binaryninja::binary_view::{BinaryView, BinaryViewExt}; +use binaryninja::file_metadata::FileMetadata; +use binaryninja::metadata::{Metadata, MetadataType}; +use binaryninja::platform::Platform; +use binaryninja::rc::Ref; +use binaryninja::types::printer::TokenEscapingType; +use binaryninja::types::{CoreTypePrinter, TypeLibrary}; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum TILDumpError { + #[error("Failed to create empty BinaryView")] + ViewCreationFailed, + + #[error("Type library has no associated platforms")] + NoPlatformFound, + + #[error("Platform '{0}' not found in Binary Ninja")] + PlatformNotFound(String), + + #[error("Failed to print types from library")] + PrinterError, + + #[error("Metadata error: {0}")] + MetadataError(String), + + #[error("Unexpected metadata type for 'ordinals': {0:?}")] + UnexpectedMetadataType(MetadataType), +} + +pub struct TILDump { + /// The type libraries that are accessible to the type printer. + available_type_libs: Vec<Ref<TypeLibrary>>, +} + +impl TILDump { + pub fn new() -> Self { + Self { + available_type_libs: Vec::new(), + } + } + + pub fn with_type_libs(mut self, type_libs: Vec<Ref<TypeLibrary>>) -> Self { + self.available_type_libs = type_libs; + self + } + + pub fn dump(&self, type_lib: &TypeLibrary) -> Result<String, TILDumpError> { + let empty_file = FileMetadata::new(); + let empty_bv = BinaryView::from_data(&empty_file, &[]) + .map_err(|_| TILDumpError::ViewCreationFailed)?; + + let type_lib_plats = type_lib.platform_names(); + let platform_name = type_lib_plats + .iter() + .next() + .ok_or(TILDumpError::NoPlatformFound)?; + + let platform_name_str = platform_name.to_string(); + let platform = Platform::by_name(&platform_name_str) + .ok_or_else(|| TILDumpError::PlatformNotFound(platform_name_str))?; + + empty_bv.set_default_platform(&platform); + + for dependency in &self.available_type_libs { + empty_bv.add_type_library(dependency); + } + empty_bv.add_type_library(type_lib); + + for ty in &type_lib.named_types() { + empty_bv.import_type_library_type(ty.name, None); + } + for obj in &type_lib.named_objects() { + empty_bv.import_type_library_object(obj.name, None); + } + + let dep_sorted_types = empty_bv.dependency_sorted_types(); + let unsorted_functions = type_lib.named_objects(); + let mut all_types: Vec<_> = dep_sorted_types + .iter() + .chain(unsorted_functions.iter()) + .collect(); + all_types.sort_by_key(|t| t.name.clone()); + + let type_printer = CoreTypePrinter::default(); + let printed_types = type_printer + .print_all_types( + all_types, + &empty_bv, + 4, + TokenEscapingType::NoTokenEscapingType, + ) + .ok_or(TILDumpError::PrinterError)?; + + let mut printed_types_str = printed_types.to_string_lossy().to_string(); + printed_types_str.push_str("\n// TYPE LIBRARY INFORMATION\n"); + + let metadata_lines = type_library_metadata_to_string(type_lib)?; + printed_types_str.push_str(&metadata_lines.join("\n")); + + empty_file.close(); + Ok(printed_types_str) + } +} + +fn type_library_metadata_to_string(type_lib: &TypeLibrary) -> Result<Vec<String>, TILDumpError> { + let mut result = Vec::new(); + for alt_name in &type_lib.alternate_names() { + result.push(format!("// ALTERNATE NAME: {}", alt_name)); + } + + let mut add_ordinals = |metadata: Ref<Metadata>| -> Result<(), TILDumpError> { + if let Some(map) = metadata.get_value_store() { + let mut list = map.iter().collect::<Vec<_>>(); + list.sort_by_key(|&(key, _)| key.parse::<u64>().unwrap_or_default()); + for (key, value) in list { + result.push(format!("// ORDINAL {}: {}", key, value)); + } + } + Ok(()) + }; + + if let Some(ordinal_key) = type_lib.query_metadata("ordinals") { + match ordinal_key.get_type() { + MetadataType::StringDataType => { + let queried_key = ordinal_key.get_string().ok_or_else(|| { + TILDumpError::MetadataError("Failed to get ordinal key string".into()) + })?; + + let queried_key_str = queried_key.to_string_lossy(); + let queried_md = type_lib.query_metadata(&queried_key_str).ok_or_else(|| { + TILDumpError::MetadataError(format!( + "Failed to query metadata for key: {}", + queried_key_str + )) + })?; + + add_ordinals(queried_md)?; + } + MetadataType::KeyValueDataType => add_ordinals(ordinal_key)?, + ty => return Err(TILDumpError::UnexpectedMetadataType(ty)), + } + } + + Ok(result) +} diff --git a/plugins/bntl_utils/src/helper.rs b/plugins/bntl_utils/src/helper.rs new file mode 100644 index 00000000..3fc1a047 --- /dev/null +++ b/plugins/bntl_utils/src/helper.rs @@ -0,0 +1,45 @@ +use binaryninja::rc::Ref; +use binaryninja::types::{NamedTypeReference, Type, TypeClass, TypeLibrary}; +use std::path::Path; +use walkdir::WalkDir; + +pub fn path_to_type_libraries(path: &Path) -> Vec<Ref<TypeLibrary>> { + WalkDir::new(path) + .into_iter() + .filter_map(|e| e.ok()) + .filter(|e| e.file_type().is_file()) + .filter(|e| e.path().extension().map_or(false, |ext| ext == "bntl")) + .filter_map(|e| TypeLibrary::load_from_file(e.path())) + .collect::<Vec<_>>() +} + +pub fn visit_type_reference(ty: &Type, visit: &mut impl FnMut(&NamedTypeReference)) { + if let Some(ntr) = ty.get_named_type_reference() { + visit(&ntr); + } + match ty.type_class() { + TypeClass::StructureTypeClass => { + let structure = ty.get_structure().unwrap(); + for field in structure.members() { + visit_type_reference(&field.ty.contents, visit); + } + for base in structure.base_structures() { + visit(&base.ty); + } + } + TypeClass::PointerTypeClass => { + visit_type_reference(&ty.child_type().unwrap().contents, visit); + } + TypeClass::ArrayTypeClass => { + visit_type_reference(&ty.child_type().unwrap().contents, visit); + } + TypeClass::FunctionTypeClass => { + let params = ty.parameters().unwrap(); + for param in params { + visit_type_reference(¶m.ty.contents, visit); + } + visit_type_reference(&ty.return_value().unwrap().contents, visit); + } + _ => {} + } +} diff --git a/plugins/bntl_utils/src/lib.rs b/plugins/bntl_utils/src/lib.rs new file mode 100644 index 00000000..aad6cdcf --- /dev/null +++ b/plugins/bntl_utils/src/lib.rs @@ -0,0 +1,63 @@ +mod command; +pub mod diff; +pub mod dump; +pub mod helper; +mod merge; +pub mod process; +pub mod schema; +pub mod tbd; +pub mod url; +pub mod validate; +mod winmd; + +#[no_mangle] +#[allow(non_snake_case)] +pub extern "C" fn CorePluginInit() -> bool { + if plugin_init().is_err() { + tracing::error!("Failed to initialize BNTL Utils plug-in"); + return false; + } + true +} + +fn plugin_init() -> Result<(), ()> { + binaryninja::tracing_init!("BNTL Utils"); + + binaryninja::command::register_command( + "BNTL\\Create\\From Current View", + "Create .bntl files from the current view", + command::create::CreateFromCurrentView {}, + ); + + binaryninja::command::register_command_for_project( + "BNTL\\Create\\From Project", + "Create .bntl files from the given project", + command::create::CreateFromProject {}, + ); + + binaryninja::command::register_command( + "BNTL\\Create\\From Directory", + "Create .bntl files from the given directory", + command::create::CreateFromDirectory {}, + ); + + binaryninja::command::register_command( + "BNTL\\Diff", + "Diff two .bntl files and output the difference to a file", + command::diff::Diff {}, + ); + + binaryninja::command::register_command( + "BNTL\\Dump To Header", + "Dump a .bntl file to a header file", + command::dump::Dump {}, + ); + + binaryninja::command::register_command( + "BNTL\\Validate", + "Validate a .bntl file and report the issues", + command::validate::Validate {}, + ); + + Ok(()) +} diff --git a/plugins/bntl_utils/src/merge.rs b/plugins/bntl_utils/src/merge.rs new file mode 100644 index 00000000..454be10f --- /dev/null +++ b/plugins/bntl_utils/src/merge.rs @@ -0,0 +1,339 @@ +//! Merge multiple similar types into one, useful when deduplicating types across different type libraries. + +use binaryninja::rc::Ref; +use binaryninja::types::{ + Enumeration, EnumerationBuilder, MemberAccess, MemberScope, Structure, StructureBuilder, Type, + TypeClass, +}; +use std::cmp::max_by_key; +use std::collections::{BTreeMap, HashMap}; +use std::num::NonZeroUsize; + +/// Merges a series of types into a single [`Type`], if possible. +pub fn merge_types(types: &[Ref<Type>]) -> Option<Ref<Type>> { + let first = types.first()?.to_owned(); + types + .iter() + .skip(1) + .try_fold(first, |acc, t| merge_recursive(&acc, t)) +} + +fn merge_recursive(t1: &Type, t2: &Type) -> Option<Ref<Type>> { + // Identical types, this is what we hope happens so we can skip the expensive merge step. + if t1 == t2 { + return Some(t1.to_owned()); + } + + // TODO: Move t1.width != t2.width check up here? I don't think there is a scenerio where it is safe. + + match (t1.type_class(), t2.type_class()) { + // Void is a wildcard for us, we will pick `t2`. + (TypeClass::VoidTypeClass, _) => Some(t2.to_owned()), + // Void is a wildcard for us, we will pick `t1`. + (_, TypeClass::VoidTypeClass) => Some(t1.to_owned()), + (TypeClass::IntegerTypeClass, TypeClass::IntegerTypeClass) => { + if t1.width() != t2.width() { + return None; + } + // Use the signedness with higher confidence + let signed = max_by_key(t1.is_signed(), t2.is_signed(), |c| c.confidence); + Some(Type::int(t1.width() as usize, signed.contents)) + } + (TypeClass::FloatTypeClass, TypeClass::FloatTypeClass) => { + if t1.width() != t2.width() { + return None; + } + Some(Type::float(t1.width() as usize)) + } + (TypeClass::PointerTypeClass, TypeClass::PointerTypeClass) => { + // Recursive merge of target; fail if targets are incompatible + let target = merge_recursive(&t1.target()?.contents, &t2.target()?.contents)?; + + let is_const = max_by_key(t1.is_const(), t2.is_const(), |c| c.confidence); + let is_vol = max_by_key(t1.is_volatile(), t2.is_volatile(), |c| c.confidence); + + Some(Type::pointer_of_width( + &target, + t1.width() as usize, + is_const.contents, + is_vol.contents, + None, + )) + } + (TypeClass::ArrayTypeClass, TypeClass::ArrayTypeClass) => { + if t1.count() != t2.count() { + return None; + } + let elem = merge_recursive(&t1.element_type()?.contents, &t2.element_type()?.contents)?; + Some(Type::array(&elem, t1.count())) + } + (TypeClass::StructureTypeClass, TypeClass::StructureTypeClass) => { + let s1 = t1.get_structure()?; + let s2 = t2.get_structure()?; + let merged = merge_structures(&s1, &s2)?; + Some(Type::structure(&merged)) + } + (TypeClass::EnumerationTypeClass, TypeClass::EnumerationTypeClass) => { + let e1 = t1.get_enumeration()?; + let e2 = t2.get_enumeration()?; + let merged = merge_enumerations(&e1, &e2)?; + + let signed = max_by_key(t1.is_signed(), t2.is_signed(), |c| c.confidence); + let width = NonZeroUsize::new(t1.width() as usize)?; + Some(Type::enumeration(&merged, width, signed)) + } + // Functions, NamedTypeReferences, etc. fall through here. + // Since we checked t1 == t2 at the start, if we reach here, they are different. + _ => None, + } +} + +fn merge_structures(s1: &Structure, s2: &Structure) -> Option<Ref<Structure>> { + let mut builder = StructureBuilder::new(); + builder.alignment(s1.alignment().max(s2.alignment())); + builder.packed(s1.is_packed()); + builder.structure_type(s1.structure_type()); + builder.width(s1.width().max(s2.width())); + + // TODO: Handle base structures (man we really should have just made those regular members) + let mut members: BTreeMap<u64, (String, Ref<Type>)> = BTreeMap::new(); + let mut merge_into_map = |s: &Structure| { + for m in &s.members() { + members + .entry(m.offset) + .and_modify(|(existing_name, existing_ty)| { + // Update type if merge succeeds + if let Some(merged) = merge_recursive(existing_ty, &m.ty.contents) { + *existing_ty = merged; + } + // Name collision: Keep existing (s1/first wins), ignoring m.name + }) + .or_insert_with(|| (m.name.clone(), m.ty.contents.to_owned())); + } + }; + + merge_into_map(s1); + merge_into_map(s2); + + for (offset, (name, ty)) in members { + builder.insert( + &ty, + &name, + offset, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + } + + Some(builder.finalize()) +} + +fn merge_enumerations(e1: &Enumeration, e2: &Enumeration) -> Option<Ref<Enumeration>> { + let mut mapped_members = HashMap::new(); + for m in &e1.members() { + mapped_members.insert(m.name.clone(), m.value); + } + for m in &e2.members() { + mapped_members.insert(m.name.clone(), m.value); + } + + let mut builder = EnumerationBuilder::new(); + for (name, value) in mapped_members { + builder.insert(&name, value); + } + Some(builder.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use binaryninja::headless::Session; + + #[test] + fn test_merge_integers() { + let _session = Session::new().expect("Failed to initialize session"); + let t1 = Type::int(4, true); // int32_t + let t2 = Type::int(4, false); // uint32_t (if conf is same, first wins? or default?) + + // Construct specific confidence to test strict merging logic + // t3 is signed with 0 confidence + let t3 = Type::named_int(4, false, "weak_uint"); + // t4 is signed with 255 confidence + let t4 = Type::named_int(4, true, "strong_int"); + + let merged = merge_types(&[t3, t4]).expect("Merge failed"); + assert!(merged.is_signed().contents); // Stronger confidence should win + assert_eq!(merged.width(), 4); + } + + #[test] + fn test_merge_void_wildcard() { + let _session = Session::new().expect("Failed to initialize session"); + let t_void = Type::void(); + let t_int = Type::int(4, true); + + // Void + Int -> Int + let merged1 = merge_types(&[t_void.clone(), t_int.clone()]).unwrap(); + assert_eq!(merged1.type_class(), TypeClass::IntegerTypeClass); + + // Int + Void -> Int + let merged2 = merge_types(&[t_int, t_void]).unwrap(); + assert_eq!(merged2.type_class(), TypeClass::IntegerTypeClass); + } + + #[test] + fn test_merge_structures_union_members() { + let _session = Session::new().expect("Failed to initialize session"); + + // Struct A: { 0: int32 } + let mut b1 = StructureBuilder::new(); + b1.insert( + &Type::int(4, true), + "a", + 0, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + let s1 = Type::structure(&b1.finalize()); + + // Struct B: { 4: float } + let mut b2 = StructureBuilder::new(); + b2.insert( + &Type::float(8), + "b", + 4, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + let s2 = Type::structure(&b2.finalize()); + + let merged_ty = merge_types(&[s1, s2]).expect("Struct merge failed"); + let merged_struct = merged_ty.get_structure().unwrap(); + let members = merged_struct.members(); + + assert_eq!(members.len(), 2); + // Members are sorted by offset + assert_eq!(members[0].offset, 0); + assert_eq!( + members[0].ty.contents.type_class(), + TypeClass::IntegerTypeClass + ); + assert_eq!(members[1].offset, 4); + assert_eq!( + members[1].ty.contents.type_class(), + TypeClass::FloatTypeClass + ); + } + + #[test] + fn test_merge_structures_overlap_conflict() { + let _session = Session::new().expect("Failed to initialize session"); + + // Struct A: { 0: int32 } + let mut b1 = StructureBuilder::new(); + b1.insert( + &Type::int(4, true), + "a", + 0, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + let s1 = Type::structure(&b1.finalize()); + + // Struct B: { 0: float } -> Conflict with A + let mut b2 = StructureBuilder::new(); + b2.insert( + &Type::float(4), + "b", + 0, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + let s2 = Type::structure(&b2.finalize()); + + let merged_ty = merge_types(&[s1, s2]).expect("Struct merge failed"); + let merged_struct = merged_ty.get_structure().unwrap(); + let members = merged_struct.members(); + + // Best effort: Keep existing if incompatible. + // Since s1 was first, it keeps int32. + assert_eq!(members.len(), 1); + assert_eq!(members[0].offset, 0); + assert_eq!( + members[0].ty.contents.type_class(), + TypeClass::IntegerTypeClass + ); + } + + #[test] + fn test_merge_pointers() { + let _session = Session::new().expect("Failed to initialize session"); + // void* + let p1 = Type::pointer_of_width(&Type::void(), 4, false, false, None); + // int32* + let p2 = Type::pointer_of_width(&Type::int(4, true), 4, false, false, None); + + let merged = merge_types(&[p1, p2]).unwrap(); + assert_eq!(merged.type_class(), TypeClass::PointerTypeClass); + + let target = merged.target().unwrap(); + // void + int -> int + assert_eq!(target.contents.type_class(), TypeClass::IntegerTypeClass); + } + + #[test] + fn test_merge_structures_name_priority() { + let _session = Session::new().expect("Failed to initialize session"); + + // Struct 1: { 0: "original_name" (int) } + let mut b1 = StructureBuilder::new(); + b1.insert( + &Type::int(4, true), + "original_name", + 0, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + let s1 = Type::structure(&b1.finalize()); + + // Struct 2: { 0: "conflict_name" (int), 4: "new_field" (int) } + let mut b2 = StructureBuilder::new(); + b2.insert( + &Type::int(4, true), + "conflict_name", // Should be ignored in favor of "original_name" + 0, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + b2.insert( + &Type::int(4, true), + "new_field", + 4, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + let s2 = Type::structure(&b2.finalize()); + + let merged_ty = merge_types(&[s1, s2]).expect("Struct merge failed"); + let merged_struct = merged_ty.get_structure().unwrap(); + let members = merged_struct.members(); + + assert_eq!(members.len(), 2); + + // Verify offset 0 kept the name from s1 + let m0 = members.iter().find(|m| m.offset == 0).unwrap(); + assert_eq!(m0.name, "original_name"); + + // Verify offset 4 was added with its name from s2 + let m4 = members.iter().find(|m| m.offset == 4).unwrap(); + assert_eq!(m4.name, "new_field"); + } +} diff --git a/plugins/bntl_utils/src/process.rs b/plugins/bntl_utils/src/process.rs new file mode 100644 index 00000000..614e8206 --- /dev/null +++ b/plugins/bntl_utils/src/process.rs @@ -0,0 +1,1294 @@ +//! Process different types of files into Binary Ninja type libraries. + +use binaryninja::architecture::CoreArchitecture; +use dashmap::DashMap; +use std::collections::{HashMap, HashSet}; +use std::env::temp_dir; +use std::ffi::OsStr; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicBool; +use std::sync::atomic::Ordering::Relaxed; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use thiserror::Error; +use walkdir::WalkDir; + +use crate::helper::visit_type_reference; +use crate::merge::merge_types; +use crate::schema::BntlSchema; +use crate::tbd::{parse_tbd_info, TbdArchitecture}; +use crate::winmd::WindowsMetadataImporter; +use binaryninja::background_task::BackgroundTask; +use binaryninja::binary_view::{BinaryView, BinaryViewExt}; +use binaryninja::custom_binary_view::BinaryViewType; +use binaryninja::file_metadata::FileMetadata; +use binaryninja::metadata::Metadata; +use binaryninja::platform::Platform; +use binaryninja::project::file::ProjectFile; +use binaryninja::project::folder::ProjectFolder; +use binaryninja::project::Project; +use binaryninja::qualified_name::QualifiedName; +use binaryninja::rc::Ref; +use binaryninja::section::Section; +use binaryninja::types::{ + CoreTypeParser, NamedTypeReference, Type, TypeClass, TypeLibrary, TypeParser, TypeParserError, +}; +use nt_apiset::{ApiSetMap, NtApiSetError}; + +#[derive(Error, Debug)] +pub enum ProcessingError { + #[error("Binary view load error: {0}")] + BinaryViewLoad(PathBuf), + + #[error("Failed to read binary view at offset {0:?} with length {1:?}")] + BinaryViewRead(u64, usize), + + #[error("Failed to read .apiset section: {0}")] + FailedToReadApiSet(#[from] NtApiSetError), + + #[error("Failed to read file: {0}")] + FileRead(std::io::Error), + + #[error("Failed to retrieve path to project file: {0:?}")] + NoPathToProjectFile(Ref<ProjectFile>), + + #[error("Processing state has been poisoned")] + StatePoisoned, + + #[error("Processing has been cancelled")] + Cancelled, + + #[error("Skipping file: {0}")] + SkippedFile(PathBuf), + + #[error("Failed to find platform: {0}")] + PlatformNotFound(String), + + #[error("Failed to parse types: {0:?}")] + TypeParsingFailed(Vec<TypeParserError>), + + #[error("Failed to import winmd: {0}")] + WinMdFailedImport(crate::winmd::ImportError), + + #[error("Failed to parse type library: {0}")] + InvalidTypeLibrary(PathBuf), +} + +#[derive(Default, Debug)] +pub struct ProcessingState { + pub cancelled: AtomicBool, + pub files: DashMap<PathBuf, bool>, +} + +impl ProcessingState { + pub fn is_cancelled(&self) -> bool { + self.cancelled.load(Relaxed) + } + + pub fn cancel(&self) { + self.cancelled.store(true, Relaxed) + } + + pub fn files_with_state(&self, state: bool) -> usize { + self.files.iter().filter(|f| *f.value() == state).count() + } + + pub fn set_file_state(&self, path: PathBuf, state: bool) { + self.files.insert(path, state); + } + + pub fn total_files(&self) -> usize { + self.files.len() + } +} + +pub fn new_processing_state_background_thread( + task: Ref<BackgroundTask>, + state: Arc<ProcessingState>, +) { + std::thread::spawn(move || { + let start = Instant::now(); + while !task.is_finished() { + std::thread::sleep(Duration::from_millis(100)); + // Check if the user wants to cancel the processing. + if task.is_cancelled() { + state.cancel(); + } + + let total = state.total_files(); + let processed = state.files_with_state(true); + let unprocessed = state.files_with_state(false); + let completion = (processed as f64 / total as f64) * 100.0; + let elapsed = start.elapsed().as_secs_f32(); + let text = format!( + "Processing {} files... {{{}|{}}} ({:.2}%) [{:.2}s]", + total, unprocessed, processed, completion, elapsed + ); + task.set_progress_text(&text); + } + }); +} + +/// The result of running [`TypeLibProcessor`]. +#[derive(Debug, Clone)] +pub struct ProcessedData { + pub type_libraries: HashSet<Ref<TypeLibrary>>, +} + +impl ProcessedData { + pub fn new(type_libraries: Vec<Ref<TypeLibrary>>) -> Self { + Self { + type_libraries: type_libraries.into_iter().collect(), + } + } + + pub fn finalized(mut self, default_name: &str) -> Self { + self.deduplicate_types(&default_name); + // TODO: Run remap. + self.prune() + } + + /// Prune empty type libraries from the processed data. + /// + /// This is useful if you intend to save the type libraries to disk in a finalized form. + pub fn prune(self) -> Self { + let is_empty = + |tl: &TypeLibrary| tl.named_types().is_empty() && tl.named_objects().is_empty(); + let pruned_type_libraries = self + .type_libraries + .into_iter() + .filter(|tl| !is_empty(tl)) + .collect::<Vec<_>>(); + Self::new(pruned_type_libraries) + } + + /// Merges multiple [`ProcessedData`] into one, deduplicating type libraries. + /// + /// This is necessary to allow the [`TypeLibProcessor`] to operate on a wide range of formats whilst + /// also guaranteeing no collisions and valid external references. Without merging libraries with + /// identical dependency names would be separate, which is not a supported scenario when loading + /// type libraries into Binary Ninja. + pub fn merge(list: &[ProcessedData]) -> Self { + let mut type_libraries = Vec::new(); + for data in list { + type_libraries.extend(data.type_libraries.iter().cloned()); + } + + // We merge type libraries with the same dependency name, as that is what needs to be unique + // when we go to load them into Binary Ninja. + let mut mapped_type_libraries: HashMap<(String, CoreArchitecture), Vec<Ref<TypeLibrary>>> = + HashMap::new(); + for tl in type_libraries.iter() { + mapped_type_libraries + .entry((tl.dependency_name(), tl.arch())) + .or_default() + .push(tl.clone()); + } + + let mut merged_type_libraries = Vec::new(); + for ((dependency_name, arch), type_libraries) in mapped_type_libraries { + // Skip the more expensive merging if there is only a single type library. + if type_libraries.len() == 1 { + merged_type_libraries.push(type_libraries[0].clone()); + continue; + } + + let merged_type_library = TypeLibrary::new(arch, &dependency_name); + merged_type_library.set_dependency_name(&dependency_name); + for tl in type_libraries { + // TODO: Cheap type overrides (if one type is set as void* and the other as Foo* we take Foo*) + for named_type in &tl.named_types() { + merged_type_library.add_named_type(named_type.name.clone(), &named_type.ty); + } + for named_object in &tl.named_objects() { + merged_type_library + .add_named_object(named_object.name.clone(), &named_object.ty); + } + for alt_name in &tl.alternate_names() { + merged_type_library.add_alternate_name(alt_name); + } + for platform_name in &tl.platform_names() { + if let Some(platform) = Platform::by_name(&platform_name) { + merged_type_library.add_platform(&platform); + } else { + // TODO: Upgrade this to an error? + tracing::warn!( + "Unknown platform name when merging '{}': '{}'", + dependency_name, + platform_name + ); + } + } + // TODO: Stealing the type sources is literally impossible there is no getter, incredible... + // TODO: Replace this with a getter to type sources :/ + let tmp_file = temp_dir().join(format!("{}_{}.json", dependency_name, tl.guid())); + if tl.decompress_to_file(&tmp_file) { + let schema = BntlSchema::from_path(&tmp_file); + for type_source in schema.type_sources { + merged_type_library + .add_type_source(type_source.name.into(), &type_source.source); + } + } + + // Merge type library metadata, which can contain ordinal mappings. + if let Some(metadata_kv) = tl.metadata().get_value_store() { + // TODO: Handle merging of inner key values. + for (key, value) in metadata_kv { + let _ = merged_type_library.metadata().insert(&key, &value); + } + } + } + merged_type_libraries.push(merged_type_library); + } + + Self::new(merged_type_libraries) + } + + /// Maps the default type library objects into their locatable type libraries, if available. + /// + /// This process is necessary only when the source of the processed data could not determine, + /// like in the case of header files, where the type library dependency name (e.g. "sqlite3.dll") + /// cannot be determined in a vacuum. + /// + /// In the absence of that dependency name, the processor also parses auxiliary information like + /// apples TBD (text-based dylib stubs) to find out where to relocate those objects. For more + /// information see [`TypeLibProcessor::process_tbd`] + pub fn remap(&mut self, default_type_library: &str) { + let Some(default_type_library) = self + .type_libraries + .iter() + .find(|tl| tl.name() == default_type_library) + else { + tracing::error!( + "Default type library '{}' not found in processed data", + default_type_library + ); + return; + }; + + // Go through all named objects and search for that symbol in another type library, if + // we find a match relocate the object. To relocate, we delete the object from the default + // type library and conditionally swap the type (to whichever is not void), recording + // visible referenced types to later relocate as well. + let mut recorded_references: HashMap<QualifiedName, HashSet<Ref<TypeLibrary>>> = + HashMap::new(); + for tl in &self.type_libraries { + for named_object in &tl.named_objects() { + if let Some(relocated_type) = + default_type_library.get_named_object(named_object.name.clone()) + { + // Move the type over to the target type library. + if named_object.ty.type_class() == TypeClass::VoidTypeClass { + // TODO: This visit actually needs to be a + visit_type_reference(&relocated_type, &mut |ntr| { + let ntr_name = ntr.name(); + // Copy over the referenced types source library so the target type library + // can use it to resolve the reference at load time. + if let Some(type_source) = + default_type_library.get_named_type_source(ntr_name.clone()) + { + tl.add_type_source(ntr_name.clone(), &type_source) + } + // Record all referenced types that reside in the same type library, so + // we can relocate them as well, assuming no other type library also uses it. + let is_associated = default_type_library + .get_named_type(ntr_name.clone()) + .is_some(); + if is_associated { + recorded_references + .entry(ntr_name) + .or_default() + .insert(tl.clone()); + } + }); + tl.add_named_object(named_object.name.clone(), &relocated_type); + } + + // TODO: Not technically necessary because the imports are keyed off dependency name. + // Remove from the default type library. + default_type_library.remove_named_object(named_object.name.clone()); + } + } + } + + // TODO: After we have gone through the named objects and moved their types over, we need to + // TODO: enumerate all types in the default type library and determine if they should be relocated + // TODO: to the new type library. Apart of this is also calling `add_type_source(ntr_name, default_type_lib)` + // TODO: for every type that is not relocated, as we now need to tell the target type library + // TODO: that the reference is external and exists in the default type library. + + // TODO: Ugh, this needs to be a work list, we have to continue to drill down to relocate. + for (qualified_name, type_libraries) in recorded_references { + if type_libraries.len() == 1 { + // Only one type library uses this type, so we can safely relocate it. + let type_library = type_libraries.iter().next().unwrap(); + let named_ty = default_type_library + .get_named_type(qualified_name.clone()) + .unwrap(); + type_library.add_named_type(qualified_name, &named_ty); + } + } + } + + /// Locates named types which exist in multiple distinct type libraries and merges them into + /// a single type library (default type library). + /// + /// Example: `Qt5Core.dll.bndb` and `Qt5Charts.dll.bndb` both had pdb info, and both have `QObject`. + /// Assuming `QObject` is mergeable, we will merge it into the default type library. + pub fn deduplicate_types(&mut self, default_type_library_name: &str) { + let mut default_libraries = HashMap::new(); + let mut get_default_type_library = |arch: CoreArchitecture| { + default_libraries + .entry(arch) + .or_insert_with(move || TypeLibrary::new(arch, default_type_library_name)) + .to_owned() + }; + + let mut mapped_named_types: HashMap< + (QualifiedName, CoreArchitecture), + Vec<Ref<TypeLibrary>>, + > = HashMap::new(); + for merged_type_library in &self.type_libraries { + for named_type in &merged_type_library.named_types() { + mapped_named_types + .entry((named_type.name.clone(), merged_type_library.arch())) + .or_default() + .push(merged_type_library.clone()); + } + } + + for ((qualified_name, arch), type_libraries) in mapped_named_types { + if type_libraries.len() == 1 { + continue; + } + let default_type_library = get_default_type_library(arch); + + let unmerged_types: Vec<_> = type_libraries + .iter() + .filter_map(|tl| tl.get_named_type(qualified_name.clone())) + .collect(); + if let Some(merged_type) = merge_types(&unmerged_types) { + // Add the merged type to the default type library, then we need to point the type + // libraries to use this newly merged type instead of their type. + default_type_library.add_named_type(qualified_name.clone(), &merged_type); + for type_library in type_libraries { + // If the default type library does not have the platform, it will not be pulled in. + for platform_name in &type_library.platform_names() { + if let Some(platform) = Platform::by_name(&platform_name) { + default_type_library.add_platform(&platform); + } + } + + type_library.remove_named_type(qualified_name.clone()); + type_library + .add_type_source(qualified_name.clone(), &default_type_library_name); + } + } else { + // TODO: Probably demote this to debug, since they might just be disparate types. + tracing::warn!( + "Unable to merge type for duplicated name: {}", + qualified_name + ); + } + } + + // Make sure all the default type libraries are within the processed data, if not already. + for (_, default_type_library) in default_libraries { + self.type_libraries.insert(default_type_library); + } + } +} + +pub struct TypeLibProcessor { + state: Arc<ProcessingState>, + /// The Binary Ninja settings to use when analyzing the binaries. + analysis_settings: serde_json::Value, + /// The default name to use for the type library dependency name (e.g. "sqlite.dll"). + /// + /// When processing information that does not contain the dependency name, this will be used, + /// such as processing header files. We need to set a dependency name, otherwise the library + /// will not be able to be referenced by other libraries and/or the binary view. + /// + /// This dependency name will NOT be used when it can otherwise be inferred by the processing + /// data, if you wish to override the resulting dependency name, you can do so by calling + /// [`TypeLibrary::set_dependency_name`] on the libraries returned via [`ProcessedData::type_libraries`]. + default_dependency_name: String, + /// The default platform name to use when processing (e.g. "windows-x86_64"). + /// + /// When processing information that does not have an associated platform, this will be used, + /// such as processing header files or processing winmd files. When processing binary files, + /// the platform will be derived from the binary view default platform. + /// + /// For WINMD files you typically want to run the processor for each of the following platforms: + /// + /// - "windows-x86_64" + /// - "windows-x86" + /// - "windows-aarch64" + default_platform_name: String, + /// Set the include directories to use when processing header files. These will be passed to the + /// Clang type parser, which will use them to resolve header file includes. + include_directories: Vec<PathBuf>, + /// Whether to process existing type libraries when processing a binary file. + process_existing_type_libraries: bool, +} + +impl TypeLibProcessor { + pub fn new(default_dependency_name: &str, default_platform_name: &str) -> Self { + Self { + state: Arc::new(ProcessingState::default()), + analysis_settings: serde_json::json!({ + "analysis.linearSweep.autorun": false, + "analysis.mode": "full", + }), + default_dependency_name: default_dependency_name.to_owned(), + default_platform_name: default_platform_name.to_owned(), + include_directories: Vec::new(), + process_existing_type_libraries: false, + } + } + + /// Retrieve a thread-safe shared reference to the [`ProcessingState`]. + pub fn state(&self) -> Arc<ProcessingState> { + self.state.clone() + } + + pub fn with_include_directories(mut self, include_directories: Vec<PathBuf>) -> Self { + self.include_directories = include_directories; + self + } + + /// Whether to process existing type libraries when processing a binary file. + /// + /// If you open `mymodule.dll` and it imports functions from `kernel32.dll`, any import found + /// within the associated `kernel32.dll.bntl` will not be processed if this is `true`. + pub fn process_existing_type_libraries( + mut self, + process_existing_type_libraries: bool, + ) -> Self { + self.process_existing_type_libraries = process_existing_type_libraries; + self + } + + /// Place a call to this in places to interrupt when canceled. + fn check_cancelled(&self) -> Result<(), ProcessingError> { + match self.state.is_cancelled() { + true => Err(ProcessingError::Cancelled), + false => Ok(()), + } + } + + pub fn process(&self, path: &Path) -> Result<ProcessedData, ProcessingError> { + match path.extension() { + Some(ext) if ext == "bntl" => self.process_type_library(&path), + Some(ext) if ext == "h" || ext == "hpp" => self.process_source(path), + // NOTE: A typical processor will not go down this path where we only provide a single + // winmd file to be processed. You almost always want to process multiple winmd files, + // which can be done by passing a directory with the relevant winmd files. + Some(ext) if ext == "winmd" => self.process_winmd(&[path.to_owned()]), + Some(ext) if ext == "tbd" => self.process_tbd(path), + _ if path.is_dir() => self.process_directory(path), + _ => self.process_file(path), + } + } + + pub fn process_directory(&self, path: &Path) -> Result<ProcessedData, ProcessingError> { + // Collect all files in the directory + let files = WalkDir::new(path) + .into_iter() + .filter_map(|e| { + let path = e.ok()?.into_path(); + if path.is_file() { + Some(path) + } else { + None + } + }) + .collect::<Vec<_>>(); + + // TODO: Parallel processing of files? + let unmerged_data: Result<Vec<_>, _> = files + .iter() + .map(|file| { + self.check_cancelled()?; + self.process(file) + }) + .filter_map(|res| match res { + Ok(result) => Some(Ok(result)), + Err(ProcessingError::SkippedFile(path)) => { + tracing::debug!("Skipping directory file: {:?}", path); + None + } + Err(ProcessingError::Cancelled) => Some(Err(ProcessingError::Cancelled)), + Err(e) => { + tracing::error!("Directory file processing error: {:?}", e); + None + } + }) + .collect(); + + Ok(ProcessedData::merge(&unmerged_data?)) + } + + pub fn process_project(&self, project: &Project) -> Result<ProcessedData, ProcessingError> { + // Inform the state of the new unprocessed project files. + for project_file in &project.files() { + // NOTE: We use the on disk path here because the downstream file state uses that. + if let Some(path) = project_file.path_on_disk() { + self.state.set_file_state(path, false); + } + } + + let data: Result<Vec<_>, _> = project + .files() + .iter() + .map(|file| { + self.check_cancelled()?; + self.process_project_file(&file) + }) + .filter_map(|res| match res { + Ok(result) => Some(Ok(result)), + Err(ProcessingError::SkippedFile(path)) => { + tracing::debug!("Skipping project root file: {:?}", path); + None + } + Err(ProcessingError::Cancelled) => Some(Err(ProcessingError::Cancelled)), + Err(e) => { + tracing::error!("Project root file processing error: {:?}", e); + None + } + }) + .collect(); + + Ok(ProcessedData::merge(&data?)) + } + + pub fn process_project_folder( + &self, + project_folder: &ProjectFolder, + ) -> Result<ProcessedData, ProcessingError> { + for project_file in &project_folder.files() { + // NOTE: We use the on disk path here because the downstream file state uses that. + if let Some(path) = project_file.path_on_disk() { + self.state.set_file_state(path, false); + } + } + + let unmerged_data: Result<Vec<_>, _> = project_folder + .files() + .iter() + .map(|file| { + self.check_cancelled()?; + self.process_project_file(&file) + }) + .filter_map(|res| match res { + Ok(result) => Some(Ok(result)), + Err(ProcessingError::SkippedFile(path)) => { + tracing::debug!("Skipping project directory file: {:?}", path); + None + } + Err(ProcessingError::Cancelled) => Some(Err(ProcessingError::Cancelled)), + Err(e) => { + tracing::error!("Project folder file processing error: {:?}", e); + None + } + }) + .collect(); + + Ok(ProcessedData::merge(&unmerged_data?)) + } + + pub fn process_project_file( + &self, + project_file: &ProjectFile, + ) -> Result<ProcessedData, ProcessingError> { + let file_name = project_file.name(); + let extension = file_name.split('.').last(); + let path = project_file + .path_on_disk() + .ok_or_else(|| ProcessingError::NoPathToProjectFile(project_file.to_owned()))?; + match extension { + Some(ext) if ext == "bntl" => self.process_type_library(&path), + Some(ext) if ext == "h" || ext == "hpp" => self.process_source(&path), + // NOTE: A typical processor will not go down this path where we only provide a single + // winmd file to be processed. You almost always want to process multiple winmd files, + // which can be done by passing a directory with the relevant winmd files. + Some(ext) if ext == "winmd" => self.process_winmd(&[path]), + Some(ext) if ext == "tbd" => self.process_tbd(&path), + _ => { + // If the file cannot be parsed, it should be skipped to avoid a load error. + if !is_parsable(&path) { + return Err(ProcessingError::SkippedFile(path.to_owned())); + } + + let settings_str = self.analysis_settings.to_string(); + let file = binaryninja::load_project_file_with_progress( + &project_file, + false, + Some(settings_str), + |_pos, _total| { + // TODO: Report progress + true + }, + ) + .ok_or_else(|| ProcessingError::BinaryViewLoad(path.to_owned()))?; + let data = self.process_view(path.to_owned(), &file); + file.file().close(); + data + } + } + } + + // TODO: Process mapping file + // TODO: A json file that maps type names to their type dlls + // TODO: Apples format (tbd to move symbols from default type lib to their actual place) + + /// NOTE: Never pass a project file into this function, use [`TypeLibProcessor::process_project_file`] + /// instead as the file metadata will not attach to the project file to the view otherwise, leading + /// to incorrect dependency names. + pub fn process_file(&self, path: &Path) -> Result<ProcessedData, ProcessingError> { + // If the file cannot be parsed, it should be skipped to avoid a load error. + if !is_parsable(path) { + return Err(ProcessingError::SkippedFile(path.to_owned())); + } + + let settings_str = self.analysis_settings.to_string(); + let file = binaryninja::load_with_options_and_progress( + &path, + false, + Some(settings_str), + |_pos, _total| { + // TODO: Report progress + true + }, + ) + .ok_or_else(|| ProcessingError::BinaryViewLoad(path.to_owned()))?; + let data = self.process_view(path.to_owned(), &file); + file.file().close(); + data + } + + pub fn process_view( + &self, + path: PathBuf, + view: &BinaryView, + ) -> Result<ProcessedData, ProcessingError> { + self.state.set_file_state(path.to_owned(), false); + let view_platform = view.default_platform().unwrap_or(self.default_platform()?); + + // Try and get the original file name, if not fall back to the default dependency name. + // TODO: I give up trying to actually make this reasonable, in the future we need to revisit + // TODO: how we save this information in the core so that its not a dozen lines of code to get + let dependency_name = match view.file().project_file() { + Some(project) => { + // We have to strip the .bndb extension because the project file path on disk is a guid + // so we just grab the project files "display name", because view.file().display_name() + // does not actually do what we want. We for some reason in the core rewrite the file + // name and display the name to be that of the bndb path instead of the file name associated + // with the actual view (which is actually useful information). + project + .path_in_project() + .file_name() + .unwrap_or(OsStr::new(&self.default_dependency_name)) + .to_string_lossy() + .strip_suffix(".bndb") + .unwrap_or(&self.default_dependency_name) + .to_string() + } + None => view + .file() + .original_file_path() + .unwrap_or(path.clone()) + .file_name() + .map(|s| s.to_string_lossy().to_string()) + .unwrap_or_else(|| self.default_dependency_name.clone()), + }; + + let type_library = TypeLibrary::new(view_platform.arch(), &dependency_name); + type_library.add_platform(&view_platform); + + // TODO: This has to be extremely slow + let platform_types = view_platform + .types() + .iter() + .map(|t| t.name.clone()) + .collect::<HashSet<_>>(); + let mut type_name_to_library = HashMap::new(); + for tl in view.type_libraries().iter() { + let lib_name = tl.name().to_string(); + for t in tl.named_types().iter() { + type_name_to_library.insert(t.name.clone(), lib_name.clone()); + } + } + + let add_referenced_types = |type_library: &TypeLibrary, ty: &Type| { + let mut referenced_ntrs: Vec<Ref<NamedTypeReference>> = Vec::new(); + visit_type_reference(ty, &mut |ntr| { + referenced_ntrs.push(ntr.to_owned()); + }); + + // Pull in all referenced types recursively. + while let Some(ntr) = referenced_ntrs.pop() { + let referenced_name = ntr.name(); + if type_library + .get_named_type(referenced_name.clone()) + .is_some() + { + continue; + } + if platform_types.contains(&referenced_name) { + // The type referenced comes from the platform, so we do not need to do anything. + } else if let Some(source) = type_name_to_library.get(&referenced_name) { + type_library.add_type_source(referenced_name, source); + } else { + // Type does not belong to another type library, so we add it to the current one. + if let Some(referenced_ty) = view.type_by_ref(&ntr) { + visit_type_reference(&referenced_ty, &mut |ntr| { + referenced_ntrs.push(ntr.to_owned()); + }); + type_library.add_named_type(referenced_name, &referenced_ty); + } else { + tracing::debug!( + "Type '{}' referenced by '{}' not found in view, skipping...", + referenced_name, + ty + ); + } + } + } + }; + + let mut ordinals: HashMap<String, String> = HashMap::new(); + let functions = view.functions(); + tracing::info!("Adding {} functions", functions.len()); + for func in &functions { + if !func.is_exported() { + continue; + } + let Some(defined_symbol) = func.defined_symbol() else { + tracing::debug!( + "Function '{}' has no defined symbol, skipping...", + func.symbol() + ); + continue; + }; + // Common case where we attach a "j_" prefix to the exported name, which ruins the symbol + // since it's expected to be imported by name. https://github.com/Vector35/binaryninja-api/issues/7970 + let name = defined_symbol + .raw_name() + .to_string_lossy() + .replace("j_", ""); + let qualified_name = QualifiedName::from(name.clone()); + type_library.add_named_object(qualified_name, &func.function_type()); + add_referenced_types(&type_library, &func.function_type()); + + if let Some(ordinal) = defined_symbol.ordinal() { + ordinals.insert(ordinal.to_string(), name); + } + } + + if !ordinals.is_empty() { + tracing::info!( + "Found {} ordinals in '{}', adding metadata...", + ordinals.len(), + view.file(), + ); + // TODO: The ordinal version is OSMAJOR_OSMINOR, pull from pe metadata (use object crate) + let key_md: Ref<Metadata> = String::from("ordinals_10_0").into(); + type_library.store_metadata("ordinals", &key_md); + let map_md: Ref<Metadata> = ordinals.into(); + type_library.store_metadata("ordinals_10_0", &map_md); + } + + let mut processed_data = self.process_external_libraries(&view)?; + processed_data.type_libraries.insert(type_library); + if let Some(api_set_section) = view.section_by_name(".apiset") { + let processed_api_set = self.process_api_set(&view, &api_set_section)?; + tracing::info!( + "Found {} api set libraries in '{}', adding alternative names...", + processed_api_set.type_libraries.len(), + view.file(), + ); + processed_data = ProcessedData::merge(&[processed_data, processed_api_set]); + } + self.state.set_file_state(path.to_owned(), true); + Ok(processed_data) + } + + pub fn process_external_libraries( + &self, + view: &BinaryView, + ) -> Result<ProcessedData, ProcessingError> { + let view_platform = view.default_platform().unwrap_or(self.default_platform()?); + let mut extern_type_libraries = HashMap::new(); + for extern_lib in &view.external_libraries() { + let extern_type_library = TypeLibrary::new(view_platform.arch(), &extern_lib.name()); + extern_type_library.add_platform(&view_platform); + extern_type_library.set_dependency_name(&extern_lib.name()); + extern_type_libraries.insert(extern_lib.name(), extern_type_library); + } + + // Pull import types and add them to respective type libraries. + for extern_loc in &view.external_locations() { + // The source symbol represents the symbol represented in the binary, while the target + // symbol represents the symbol that we intend to map the information to. + let src_sym = extern_loc.source_symbol(); + let Some(extern_lib) = extern_loc.library() else { + tracing::debug!( + "External location '{}' has no library, skipping...", + src_sym + ); + continue; + }; + let Some(extern_type_library) = extern_type_libraries.get_mut(&extern_lib.name()) + else { + tracing::warn!( + "External location '{}' is referencing a detached external library, skipping...", + src_sym + ); + continue; + }; + let Some(src_data_var) = view.data_variable_at_address(src_sym.address()) else { + tracing::debug!( + "External location '{}' has no data variable, skipping...", + src_sym + ); + continue; + }; + if src_data_var.auto_discovered { + // We do not want to record objects which are not modified by the user, otherwise + // we are recording the object each time we visit a binary view, possibly retrieving + // the old definition of the object. + tracing::debug!( + "External location '{}' is auto discovered, skipping...", + src_sym + ); + continue; + } + let target_sym_name = extern_loc + .target_symbol() + .unwrap_or_else(|| src_sym.raw_name()); + + if !self.process_existing_type_libraries + && view + .import_type_library_object(target_sym_name.clone(), None) + .is_some() + { + tracing::debug!( + "Skipping external location '{}' as it is already present in a type library", + target_sym_name.to_string_lossy() + ); + continue; + } + + // TODO: Need to visit all types referenced and add it to the type library. + extern_type_library.add_named_object(target_sym_name.into(), &src_data_var.ty.contents); + } + + Ok(ProcessedData::new( + extern_type_libraries.values().cloned().collect(), + )) + } + + /// Process API sets on Windows binaries, so we can fill in the alternative names for type libraries + /// we are processing. + /// + /// Creates an empty type library for the host and adds the alternative names to it. This should then + /// be passed to the [`ProcessedData::merge`] set to be merged with the type library of the host name. + /// + /// For more information see: https://learn.microsoft.com/en-us/windows/win32/apiindex/windows-apisets + pub fn process_api_set( + &self, + view: &BinaryView, + section: &Section, + ) -> Result<ProcessedData, ProcessingError> { + let section_bytes = view + .read_buffer(section.start(), section.len()) + .ok_or_else(|| ProcessingError::BinaryViewRead(section.start(), section.len()))?; + let api_set_map = ApiSetMap::try_from_apiset_section_bytes(§ion_bytes.get_data())?; + + let mut target_map: HashMap<String, HashSet<String>> = HashMap::new(); + for entry in api_set_map.namespace_entries()? { + let alternative_name = entry.name()?.to_string_lossy(); + for value_entry in entry.value_entries()? { + // TODO: In cases where alt -> kernel32.dll -> kernelbase.dll we currently associate + // TODO: with kernel32.dll as its assumed there is a wrapper function that calls into + // TODO: kernelbase.dll. This keeps us from having to validate against both, in the case + // TODO: of kernelbase.dll being before the function was moved there. + let _forwarder_name = value_entry.name()?.to_string_lossy(); + let target_name = value_entry.value()?.to_string_lossy(); + target_map + .entry(target_name) + .or_default() + .insert(alternative_name.clone()); + } + } + + // Instead of using the view, we use the user-provided platform, the reason is because the + // 'apisetschema.dll' is shared across multiple archs, and we need to be able to merge its data + // with other platforms so that they get the correct alternative names. + let platform = self.default_platform()?; + let mut mapping_type_libraries = Vec::new(); + for (target_name, alternative_names) in target_map { + let type_library = TypeLibrary::new(platform.arch(), &target_name); + for alt_name in alternative_names { + type_library.add_alternate_name(&alt_name); + } + mapping_type_libraries.push(type_library); + } + + Ok(ProcessedData::new(mapping_type_libraries)) + } + + /// We want to be able to process already created type libraries so that they can be consulted + /// during the [`ProcessedData::merge`] step. This lets us add overrides like extra platforms. + pub fn process_type_library(&self, path: &Path) -> Result<ProcessedData, ProcessingError> { + self.state.set_file_state(path.to_owned(), false); + let finalized_type_library = TypeLibrary::load_from_file(&path) + .ok_or_else(|| ProcessingError::InvalidTypeLibrary(path.to_owned()))?; + self.state.set_file_state(path.to_owned(), true); + Ok(ProcessedData::new(vec![finalized_type_library])) + } + + pub fn process_source(&self, path: &Path) -> Result<ProcessedData, ProcessingError> { + self.state.set_file_state(path.to_owned(), false); + let platform = self.default_platform()?; + let parser = + CoreTypeParser::parser_by_name("ClangTypeParser").expect("Failed to get clang parser"); + let platform_type_container = platform.type_container(); + + let header_contents = + std::fs::read_to_string(path).map_err(|e| ProcessingError::FileRead(e))?; + + let file_name = path + .file_name() + .unwrap_or(OsStr::new("source.hpp")) + .to_string_lossy(); + // TODO: Allow specifying options? + let mut include_dirs = self.include_directories.clone(); + // TODO: This will not work for projects, we need to remove this parent call + // TODO: and place it in the `include_directories`, where we parse that from the user input + // TODO: To the + if let Some(p) = path.parent() { + include_dirs.push(p.to_owned()); + } + let parsed_types = parser + .parse_types_from_source( + &header_contents, + &file_name, + &platform, + &platform_type_container, + &[], + &include_dirs, + "", + ) + .map_err(|e| ProcessingError::TypeParsingFailed(e))?; + + let type_library = TypeLibrary::new(platform.arch(), &self.default_dependency_name); + type_library.add_platform(&platform); + for ty in parsed_types.types { + type_library.add_named_type(ty.name, &ty.ty); + } + for func in parsed_types.functions { + type_library.add_named_object(func.name, &func.ty); + } + self.state.set_file_state(path.to_owned(), true); + Ok(ProcessedData::new(vec![type_library])) + } + + /// Processes Apples TBD file format, which is a YAML file that contains information about a dylib, + /// most important for us is the list of exported symbols, which we can use to relocate objects + /// in the default type library (specified by `default_dependency_name`) to the correct type library. + pub fn process_tbd(&self, path: &Path) -> Result<ProcessedData, ProcessingError> { + let mut file = File::open(path).map_err(|e| ProcessingError::FileRead(e))?; + let mut type_libraries = Vec::new(); + for tbd_info in parse_tbd_info(&mut file).unwrap() { + let install_path = PathBuf::from(tbd_info.install_name); + let library_name = install_path + .file_name() + .unwrap() + .to_string_lossy() + .to_string(); + + let mut mapped_type_libraries: HashMap<TbdArchitecture, Vec<Ref<TypeLibrary>>> = + HashMap::new(); + for target in tbd_info.targets { + let Some(target_platform) = target.binary_ninja_platform() else { + tracing::error!( + "Failed to find platform '{:?}' when parsing file: {}", + target, + path.display() + ); + continue; + }; + let type_library = TypeLibrary::new(target_platform.arch(), &library_name); + type_library.add_platform(&target_platform); + mapped_type_libraries + .entry(target.arch) + .or_default() + .push(type_library); + } + + for exports in tbd_info.exports { + for export_target in exports.targets { + let type_libraries = mapped_type_libraries.get(&export_target.arch).unwrap(); + for type_library in type_libraries { + // TODO: Handle `objc_classes`? + for symbol in &exports.symbols { + // TODO: We need more than just a void type to differentiate, because we have + // TODO: NTRs that might be pointing to void types, i dont really want to + // TODO: detach the backing type there. + // Create a void type for the symbol, we prune these at the end, so if + // nothing remaps to this library with the given symbol, it will go away. + type_library.add_named_object(symbol.into(), &Type::void()); + } + } + } + } + + for (_, libs) in mapped_type_libraries { + type_libraries.extend(libs); + } + } + + Ok(ProcessedData::new(type_libraries)) + } + + /// Unlike [`TypeLibProcessor::process_source`] which can pass include directories, this processing + /// requires us to actually load multiple files to parse the correct information. + /// + /// A specific example of this is the "Windows.Wdk.winmd" references types in "Windows.Win32.winmd". + /// If we did not process them together, we would have unresolved references when loading kernel + /// type libraries. + pub fn process_winmd(&self, paths: &[PathBuf]) -> Result<ProcessedData, ProcessingError> { + for path in paths { + self.state.set_file_state(path.to_owned(), false); + } + let platform = self.default_platform()?; + let type_libraries = WindowsMetadataImporter::new() + .with_files(&paths) + .map_err(ProcessingError::WinMdFailedImport)? + .import(&platform) + .map_err(ProcessingError::WinMdFailedImport)?; + for path in paths { + self.state.set_file_state(path.to_owned(), true); + } + Ok(ProcessedData::new(type_libraries)) + } + + pub fn default_platform(&self) -> Result<Ref<Platform>, ProcessingError> { + Platform::by_name(&self.default_platform_name) + .ok_or_else(|| ProcessingError::PlatformNotFound(self.default_platform_name.clone())) + } +} + +pub fn is_parsable(path: &Path) -> bool { + if binaryninja::is_database(path) { + return true; + } + // For some reason these pass to a view type? + if path.extension() == Some(OsStr::new("pdb")) { + return false; + } + let mut metadata = FileMetadata::with_file_path(&path); + let Ok(view) = BinaryView::from_path(&mut metadata, path) else { + return false; + }; + // If any view type parses this file, consider it for this source. + // All files will have a "Raw" file type, so we account for that. + BinaryViewType::list_valid_types_for(&view).len() > 1 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_is_parsable() { + let _session = + binaryninja::headless::Session::new().expect("Failed to create headless session"); + let data_dir = Path::new(&env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("Cargo workspace directory") + .join("data"); + let x86_file_path = data_dir.join("x86").join("mfc42.dll.bndb"); + assert!(x86_file_path.exists()); + assert!(is_parsable(&x86_file_path)); + let header_file_path = data_dir.join("headers").join("test.h"); + assert!(header_file_path.exists()); + assert!(!is_parsable(&header_file_path)); + } + + #[test] + fn test_process_winmd() { + let _session = + binaryninja::headless::Session::new().expect("Failed to create headless session"); + let data_dir = Path::new(&env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("Cargo workspace directory") + .join("data"); + let win32_winmd_path = data_dir.join("winmd").join("Windows.Win32.winmd"); + assert!(win32_winmd_path.exists()); + let wdk_winmd_path = data_dir.join("winmd").join("Windows.Wdk.winmd"); + assert!(wdk_winmd_path.exists()); + + let processor = TypeLibProcessor::new("foo", "windows-x86_64"); + let processed_data = processor + .process_winmd(&[win32_winmd_path, wdk_winmd_path]) + .expect("Failed to process winmd"); + assert_eq!(processed_data.type_libraries.len(), 591); + + // Make sure processing a directory will correctly group winmd files. + let processed_folder_data = processor + .process_directory(&data_dir.join("winmd")) + .expect("Failed to process directory"); + assert_eq!(processed_folder_data.type_libraries.len(), 591); + } + + #[test] + fn test_process_source() { + let _session = + binaryninja::headless::Session::new().expect("Failed to create headless session"); + let data_dir = Path::new(&env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("Cargo workspace directory") + .join("data"); + let header_file_path = data_dir.join("headers").join("test.h"); + assert!(header_file_path.exists()); + + let processor = TypeLibProcessor::new("test.dll", "windows-x86_64"); + let processed_data = processor + .process_source(&header_file_path) + .expect("Failed to process source"); + assert_eq!(processed_data.type_libraries.len(), 1); + let processed_library = &processed_data.type_libraries.iter().next().unwrap(); + assert_eq!(processed_library.name(), "test.dll"); + assert_eq!(processed_library.dependency_name(), "test.dll"); + assert_eq!( + processed_library.platform_names().to_vec(), + vec!["windows-x86_64"] + ); + + processed_library + .get_named_type("MyStruct".into()) + .expect("Failed to get type"); + + // Make sure includes are pulled into the type library. + let header2_file_path = data_dir.join("headers").join("test2.hpp"); + let processed_data_2 = processor + .process_source(&header2_file_path) + .expect("Failed to process source"); + assert_eq!(processed_data_2.type_libraries.len(), 1); + let processed_library_2 = &processed_data_2.type_libraries.iter().next().unwrap(); + assert_eq!(processed_library_2.name(), "test.dll"); + assert_eq!(processed_library_2.dependency_name(), "test.dll"); + assert_eq!( + processed_library_2.platform_names().to_vec(), + vec!["windows-x86_64"] + ); + processed_library_2 + .get_named_type("MyStruct2".into()) + .expect("Failed to get type"); + processed_library_2 + .get_named_type("MyStruct".into()) + .expect("Failed to get included type"); + } + + #[test] + fn test_process_file() { + let _session = + binaryninja::headless::Session::new().expect("Failed to create headless session"); + let data_dir = Path::new(&env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("Cargo workspace directory") + .join("data"); + let x86_file_path = data_dir.join("x86_64").join("mfc42.dll.bndb"); + assert!(x86_file_path.exists()); + let processor = TypeLibProcessor::new("mfc42.dll", "windows-x86_64"); + let processed_data = processor + .process_file(&x86_file_path) + .expect("Failed to process file"); + assert_eq!(processed_data.type_libraries.len(), 27); + let processed_library = processed_data + .type_libraries + .iter() + .find(|lib| lib.name() == "mfc42.dll") + .expect("Failed to find mfc42.dll library"); + assert_eq!(processed_library.name(), "mfc42.dll"); + assert_eq!(processed_library.dependency_name(), "mfc42.dll"); + assert_eq!( + processed_library.platform_names().to_vec(), + vec!["windows-x86_64"] + ); + } + + #[test] + fn test_process_api_set() { + let _session = + binaryninja::headless::Session::new().expect("Failed to create headless session"); + let data_dir = Path::new(&env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("Cargo workspace directory") + .join("data"); + let apiset_file_path = data_dir.join("apiset").join("apisetschema.dll"); + assert!(apiset_file_path.exists()); + let processor = TypeLibProcessor::new("foo", "windows-x86_64"); + let processed_data = processor + .process_file(&apiset_file_path) + .expect("Failed to process file"); + + assert_eq!(processed_data.type_libraries.len(), 287); + let combase_library = processed_data + .type_libraries + .iter() + .find(|tl| tl.name() == "combase.dll") + .expect("Failed to find combase.dll type library"); + assert_eq!( + combase_library.alternate_names().to_vec(), + vec![ + "api-ms-win-core-com-l1-1-3", + "api-ms-win-core-com-midlproxystub-l1-1-0", + "api-ms-win-core-com-private-l1-1-1", + "api-ms-win-core-com-private-l1-2-0", + "api-ms-win-core-com-private-l1-3-1", + "api-ms-win-core-marshal-l1-1-0", + "api-ms-win-core-winrt-error-l1-1-1", + "api-ms-win-core-winrt-errorprivate-l1-1-1", + "api-ms-win-core-winrt-l1-1-0", + "api-ms-win-core-winrt-registration-l1-1-0", + "api-ms-win-core-winrt-roparameterizediid-l1-1-0", + "api-ms-win-core-winrt-string-l1-1-1", + "api-ms-win-downlevel-ole32-l1-1-0" + ] + ); + } + + #[test] + fn test_data_merging() { + let _session = + binaryninja::headless::Session::new().expect("Failed to create headless session"); + let x86_platform = Platform::by_name("x86").expect("Failed to get x86 platform"); + let x86_windows_platform = + Platform::by_name("windows-x86").expect("Failed to get windows x86 platform"); + // Make two type libraries with the same name, but different dependencies. + let tl1 = TypeLibrary::new(x86_platform.arch(), "foo"); + tl1.set_dependency_name("foo"); + tl1.add_platform(&x86_platform); + tl1.add_named_type("bar".into(), &Type::named_float(3, "bla")); + let tl1_data = ProcessedData::new(vec![tl1]); + + let tl2 = TypeLibrary::new(x86_platform.arch(), "bar"); + tl2.set_dependency_name("foo"); + tl2.add_platform(&x86_windows_platform); + tl2.add_named_type("baz".into(), &Type::named_int(64, false, "fre")); + let tl2_data = ProcessedData::new(vec![tl2]); + + let merged_data = ProcessedData::merge(&[tl1_data, tl2_data]); + assert_eq!(merged_data.type_libraries.len(), 1); + let merged_tl = &merged_data.type_libraries.iter().next().unwrap(); + assert_eq!(merged_tl.name(), "foo"); + assert_eq!(merged_tl.dependency_name(), "foo"); + assert_eq!(merged_tl.platform_names().len(), 2); + assert_eq!(merged_tl.named_types().len(), 2); + } +} diff --git a/plugins/bntl_utils/src/schema.rs b/plugins/bntl_utils/src/schema.rs new file mode 100644 index 00000000..2a9e6d50 --- /dev/null +++ b/plugins/bntl_utils/src/schema.rs @@ -0,0 +1,42 @@ +use serde::Deserialize; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::path::Path; + +#[derive(Deserialize, Debug)] +pub struct BntlSchema { + // The list of library names this library depends on + pub dependencies: Vec<String>, + // Maps internal type IDs or names to their external sources + pub type_sources: Vec<TypeSource>, +} + +impl BntlSchema { + pub fn from_file(file: &File) -> Self { + serde_json::from_reader(file).expect("JSON schema mismatch") + } + + pub fn from_path(path: &Path) -> Self { + Self::from_file(&File::open(path).expect("Failed to open schema file")) + } + + pub fn to_source_map(&self) -> HashMap<String, HashSet<String>> { + let mut dependencies_map: HashMap<String, HashSet<String>> = HashMap::new(); + for ts in &self.type_sources { + let full_name = ts.name.join("::"); + dependencies_map + .entry(ts.source.clone()) + .or_default() + .insert(full_name); + } + dependencies_map + } +} + +#[derive(Deserialize, Debug)] +pub struct TypeSource { + // The components of the name, e.g., ["std", "string"] + pub name: Vec<String>, + // The name of the dependency library it comes from + pub source: String, +} diff --git a/plugins/bntl_utils/src/tbd.rs b/plugins/bntl_utils/src/tbd.rs index e69de29b..51222766 100644 --- a/plugins/bntl_utils/src/tbd.rs +++ b/plugins/bntl_utils/src/tbd.rs @@ -0,0 +1,443 @@ +use binaryninja::architecture::CoreArchitecture; +use binaryninja::platform::Platform; +use binaryninja::rc::Ref; +use serde::{Deserialize, Deserializer, Serialize}; +use std::io::Read; +use std::str::FromStr; + +pub fn parse_tbd_info(data: &mut impl Read) -> Result<Vec<TbdInfo>, serde_saphyr::Error> { + let mut documents = Vec::new(); + for file in serde_saphyr::read::<_, TbdFile>(data) { + if let Some(info) = TbdInfo::try_from(file?).ok() { + documents.push(info); + } + } + Ok(documents) +} + +#[derive(Debug)] +pub struct TbdInfo { + /// The installation name of the library. + /// + /// Ex. `/usr/lib/libSystem.B.dylib` + pub install_name: String, + pub targets: Vec<TbdTarget>, + pub exports: Vec<ExportInfo>, + pub current_version: Option<String>, + pub compatibility_version: Option<String>, +} + +#[derive(Debug)] +pub struct ExportInfo { + pub targets: Vec<TbdTarget>, + pub symbols: Vec<String>, + pub objc_classes: Vec<String>, +} + +#[derive(Debug, Serialize)] +pub enum TbdFile { + #[serde(rename = "!tapi-tbd")] + V4(TbdV4), + #[serde(rename = "!tapi-tbd-v3")] + V3(TbdLegacy), + #[serde(rename = "!tapi-tbd-v2")] + V2(TbdLegacy), + #[serde(untagged)] + V1(TbdLegacy), +} + +impl<'de> Deserialize<'de> for TbdFile { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + use serde::de::Error; + + let value: serde_json::Value = Deserialize::deserialize(deserializer)?; + // V4 requires the 'tbd-version' field, if we don't see that, then fallback to legacy. + // TODO: If v5 comes out we will need to actually read the version field. + if value.get("tbd-version").is_some() { + let v4 = TbdV4::deserialize(value).map_err(D::Error::custom)?; + Ok(TbdFile::V4(v4)) + } else if value.get("archs").is_some() && value.get("platform").is_some() { + // TODO: It would be nice to determine v2 and v3 versions, but they are backwards compatible + // TODO: so its not a higher priority (we do not differentiate between them anyways) + let legacy = TbdLegacy::deserialize(value).map_err(D::Error::custom)?; + Ok(TbdFile::V1(legacy)) + } else { + Err(D::Error::custom( + "Could not determine TBD version from tags or fields", + )) + } + } +} + +impl TryFrom<TbdFile> for TbdInfo { + type Error = String; + + fn try_from(file: TbdFile) -> Result<Self, Self::Error> { + match file { + TbdFile::V4(v4) => TbdInfo::try_from(v4), + TbdFile::V3(legacy) => TbdInfo::try_from(legacy), + TbdFile::V2(legacy) => TbdInfo::try_from(legacy), + TbdFile::V1(legacy) => TbdInfo::try_from(legacy), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct TbdV4 { + pub tbd_version: u32, + pub targets: Vec<TbdTarget>, + pub install_name: String, + #[serde(default, deserialize_with = "coerce_string_opt")] + pub current_version: Option<String>, + #[serde(default, deserialize_with = "coerce_string_opt")] + pub compatibility_version: Option<String>, + #[serde(default)] + pub swift_abi_version: Option<u32>, + #[serde(default)] + pub flags: Vec<String>, + #[serde(default)] + pub exports: Vec<ExportSectionV4>, +} + +impl TryFrom<TbdV4> for TbdInfo { + type Error = String; + + fn try_from(v4: TbdV4) -> Result<Self, Self::Error> { + Ok(TbdInfo { + install_name: v4.install_name, + targets: v4.targets, + exports: v4 + .exports + .into_iter() + .map(|e| ExportInfo { + targets: e.targets, + symbols: e.symbols, + objc_classes: e.objc_classes, + }) + .collect(), + current_version: v4.current_version, + compatibility_version: v4.compatibility_version, + }) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ExportSectionV4 { + pub targets: Vec<TbdTarget>, + #[serde(default)] + pub symbols: Vec<String>, + #[serde(default)] + pub objc_classes: Vec<String>, + #[serde(default)] + pub objc_eh_types: Vec<String>, + #[serde(default)] + pub objc_ivars: Vec<String>, +} + +/// Used for TBD files from older versions (1-3) of Xcode. +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct TbdLegacy { + pub archs: Vec<TbdArchitecture>, + pub platform: TbdPlatform, + pub install_name: String, + #[serde(default, deserialize_with = "coerce_string_opt")] + pub current_version: Option<String>, + // V1/V2 used swift-version [cite: 57, 63] + #[serde(alias = "swift-version")] + pub swift_abi_version: Option<String>, + #[serde(default)] + pub exports: Vec<ExportSectionLegacy>, +} + +impl TryFrom<TbdLegacy> for TbdInfo { + type Error = String; + + fn try_from(legacy: TbdLegacy) -> Result<Self, Self::Error> { + let mut unified_exports = Vec::new(); + for export in legacy.exports { + unified_exports.push(ExportInfo { + targets: TbdTarget::from_seperate(&export.archs, &legacy.platform)?, + symbols: export.symbols, + objc_classes: export.objc_classes, + }); + } + + Ok(TbdInfo { + install_name: legacy.install_name, + targets: TbdTarget::from_seperate(&legacy.archs, &legacy.platform)?, + exports: unified_exports, + current_version: legacy.current_version, + compatibility_version: None, + }) + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub struct ExportSectionLegacy { + pub archs: Vec<TbdArchitecture>, + #[serde(default)] + pub symbols: Vec<String>, + // V1 compatibility [cite: 57, 93] + #[serde(alias = "allowed-clients")] + pub allowable_clients: Option<Vec<String>>, + #[serde(default)] + pub objc_classes: Vec<String>, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] +#[serde(rename_all = "lowercase")] +pub enum TbdArchitecture { + I386, + X86_64, + X86_64h, + Armv7, + Armv7s, + Armv7k, + Arm64, + Arm64e, +} + +impl TbdArchitecture { + pub fn binary_ninja_architecture(&self) -> Option<CoreArchitecture> { + match self { + TbdArchitecture::I386 => CoreArchitecture::by_name("x86"), + TbdArchitecture::X86_64 => CoreArchitecture::by_name("x86_64"), + TbdArchitecture::X86_64h => CoreArchitecture::by_name("x86_64"), + TbdArchitecture::Armv7 => CoreArchitecture::by_name("armv7"), + TbdArchitecture::Armv7s => CoreArchitecture::by_name("armv7"), + TbdArchitecture::Armv7k => CoreArchitecture::by_name("armv7"), + TbdArchitecture::Arm64 => CoreArchitecture::by_name("aarch64"), + TbdArchitecture::Arm64e => CoreArchitecture::by_name("aarch64"), + } + } +} + +impl FromStr for TbdArchitecture { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s.to_lowercase().as_str() { + "i386" => Ok(TbdArchitecture::I386), + "x86_64" => Ok(TbdArchitecture::X86_64), + "x86_64h" => Ok(TbdArchitecture::X86_64h), + "armv7" => Ok(TbdArchitecture::Armv7), + "armv7s" => Ok(TbdArchitecture::Armv7s), + "armv7k" => Ok(TbdArchitecture::Armv7k), + "arm64" => Ok(TbdArchitecture::Arm64), + "arm64e" => Ok(TbdArchitecture::Arm64e), + _ => Err(format!("Unknown architecture: {}", s)), + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Hash)] +#[serde(rename_all = "lowercase")] +pub enum TbdPlatform { + Macos, + Ios, + IosSimulator, + Tvos, + TvosSimulator, + Watchos, + WatchosSimulator, + Bridgeos, + Maccatalyst, +} + +impl TbdPlatform { + pub fn binary_ninja_platform_str(&self) -> &'static str { + match self { + TbdPlatform::Macos => "mac", + TbdPlatform::Ios => "ios", + TbdPlatform::IosSimulator => "ios", + TbdPlatform::Tvos => "tvos", + TbdPlatform::TvosSimulator => "tvos", + TbdPlatform::Watchos => "watchos", + TbdPlatform::WatchosSimulator => "watchos", + TbdPlatform::Bridgeos => "bridgeos", + TbdPlatform::Maccatalyst => "mac", + } + } +} + +impl FromStr for TbdPlatform { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + match s.to_lowercase().as_str() { + "macos" | "macosx" => Ok(TbdPlatform::Macos), + "ios" => Ok(TbdPlatform::Ios), + "ios-simulator" => Ok(TbdPlatform::IosSimulator), + "tvos" => Ok(TbdPlatform::Tvos), + "tvos-simulator" => Ok(TbdPlatform::TvosSimulator), + "watchos" => Ok(TbdPlatform::Watchos), + "watchos-simulator" => Ok(TbdPlatform::WatchosSimulator), + "bridgeos" => Ok(TbdPlatform::Bridgeos), + "maccatalyst" => Ok(TbdPlatform::Maccatalyst), + _ => Err(format!("Unknown platform: {}", s)), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Hash)] +pub struct TbdTarget { + pub arch: TbdArchitecture, + pub platform: TbdPlatform, +} + +impl TbdTarget { + pub fn from_seperate( + archs: &[TbdArchitecture], + platform: &TbdPlatform, + ) -> Result<Vec<Self>, String> { + archs + .iter() + .map(|a| { + Ok(TbdTarget { + arch: a.clone(), + platform: platform.clone(), + }) + }) + .collect() + } + + pub fn binary_ninja_platform(&self) -> Option<Ref<Platform>> { + let arch = self.arch.binary_ninja_architecture()?; + let platform_str = self.platform.binary_ninja_platform_str(); + let arch_platform_str = format!("{}-{}", platform_str, arch.name()); + Platform::by_name(&arch_platform_str) + } +} + +impl FromStr for TbdTarget { + type Err = String; + + fn from_str(s: &str) -> Result<Self, Self::Err> { + let parts: Vec<&str> = s.split('-').collect(); + if parts.len() < 2 { + return Err(format!("Invalid target format: {}", s)); + } + // The first part is always the architecture + let arch = TbdArchitecture::from_str(parts[0])?; + // The remaining parts form the platform [cite: 13, 15] + let platform_str = parts[1..].join("-"); + let platform = TbdPlatform::from_str(&platform_str)?; + Ok(TbdTarget { arch, platform }) + } +} + +impl<'de> Deserialize<'de> for TbdTarget { + fn deserialize<D>(deserializer: D) -> Result<Self, D::Error> + where + D: Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + TbdTarget::from_str(&s).map_err(serde::de::Error::custom) + } +} + +fn coerce_string_opt<'de, D>(deserializer: D) -> Result<Option<String>, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + #[serde(untagged)] + enum RawValue { + String(String), + Float(f64), + Int(i64), + } + + match Option::<RawValue>::deserialize(deserializer)? { + Some(RawValue::String(s)) => Ok(Some(s)), + Some(RawValue::Float(f)) => Ok(Some(f.to_string())), + Some(RawValue::Int(i)) => Ok(Some(i.to_string())), + None => Ok(None), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + const LEGACY_TBD: &str = r#" +--- +archs: [ armv7, armv7s, arm64 ] +platform: ios +install-name: /usr/lib/libsqlite3.dylib +current-version: 216.4 +compatibility-version: 9.0 +exports: + - archs: [ armv7, armv7s, arm64 ] + symbols: [ _sqlite3VersionNumber, _sqlite3VersionString, _sqlite3_close ] +... +"#; + + const V4_TBD: &str = r#" +--- !tapi-tbd +tbd-version: 4 +targets: [ x86_64-macos, arm64-macos ] +install-name: '/System/Library/Frameworks/VideoToolbox.framework/Versions/A/VideoToolbox' +current-version: 1.0 +exports: + - targets: [ x86_64-macos, arm64-macos ] + symbols: [ _VTCompressionSessionCreate, _VTDecompressionSessionCreate ] +... +"#; + + #[test] + fn test_parse_legacy_tbd() { + let mut cursor = Cursor::new(LEGACY_TBD); + let result = parse_tbd_info(&mut cursor).expect("Should parse legacy TBD"); + assert_eq!(result.len(), 1); + let info = &result[0]; + assert_eq!(info.install_name, "/usr/lib/libsqlite3.dylib"); + assert_eq!(info.targets.len(), 3); + assert_eq!(info.targets[0].arch, TbdArchitecture::Armv7); + assert_eq!(info.targets[0].platform, TbdPlatform::Ios); + let export = &info.exports[0]; + assert!(export.symbols.contains(&"_sqlite3_close".to_string())); + } + + #[test] + fn test_parse_v4_tbd() { + let mut cursor = Cursor::new(V4_TBD); + let result = parse_tbd_info(&mut cursor).expect("Should parse V4 TBD"); + + assert_eq!(result.len(), 1); + let info = &result[0]; + assert_eq!( + info.install_name, + "/System/Library/Frameworks/VideoToolbox.framework/Versions/A/VideoToolbox" + ); + + assert_eq!(info.targets.len(), 2); + let has_arm64_macos = info + .targets + .iter() + .any(|t| t.arch == TbdArchitecture::Arm64 && t.platform == TbdPlatform::Macos); + assert!(has_arm64_macos); + } + + #[test] + fn test_multi_document_parsing() { + // TBD v3+ supports multiple YAML documents in one file + let multi_doc = format!("{}\n{}", V4_TBD, LEGACY_TBD); + let mut cursor = Cursor::new(multi_doc); + let result = parse_tbd_info(&mut cursor).expect("Should parse multiple documents"); + assert_eq!(result.len(), 2); + assert_eq!( + result[0].install_name, + "/System/Library/Frameworks/VideoToolbox.framework/Versions/A/VideoToolbox" + ); + assert_eq!(result[1].install_name, "/usr/lib/libsqlite3.dylib"); + } +} diff --git a/plugins/bntl_utils/src/templates/validate.html b/plugins/bntl_utils/src/templates/validate.html new file mode 100644 index 00000000..e37af45b --- /dev/null +++ b/plugins/bntl_utils/src/templates/validate.html @@ -0,0 +1,101 @@ +<!DOCTYPE html> +<html> +<head> + {# palette() is reading from the QT style sheet FYI #} + <style> + body { + color: palette(text); + font-family: "Segoe UI", "Ubuntu", "Helvetica Neue", sans-serif; + font-size: 13px; + margin: 20px; + line-height: 1.5; + } + .header { + border-bottom: 2px solid palette(mid); + padding-bottom: 10px; + margin-bottom: 20px; + } + .header h1 { + color: palette(highlight); + font-size: 18px; + margin: 0; + text-transform: uppercase; + letter-spacing: 1px; + } + .issue-card { + background-color: rgba(128, 128, 128, 0.1); + border-left: 4px solid #ff4444; + margin-bottom: 12px; + padding: 12px 16px; + border-radius: 2px; + } + + .issue-error { border-left-color: #ef5350; } + .issue-title { + font-weight: bold; + color: palette(window-text); + display: block; + margin-bottom: 4px; + } + .issue-detail { + color: palette(highlight); + font-family: "Consolas", "Monaco", monospace; + background-color: rgba(0, 0, 0, 0.05); + padding: 2px 4px; + border-radius: 3px; + } + .label { + color: palette(disabled-text); + font-size: 11px; + text-transform: uppercase; + margin-right: 5px; + } + </style> +</head> +<body> + +<div class="header"> + <h1>Type Library Validation Report</h1> +</div> + +{% for issue in issues %} +<div class="issue-card issue-error"> + + {% if issue.DuplicateGUID %} + <span class="issue-title">Duplicate GUID</span> + <span>The GUID <span class="issue-detail">{{ issue.DuplicateGUID.guid }}</span> is already used by <span class="issue-detail">{{ issue.DuplicateGUID.existing_library }}</span>.</span> + + {% elif issue.DuplicateDependencyName %} + <span class="issue-title">Dependency Name Collision</span> + <span>The name <span class="issue-detail">{{ issue.DuplicateDependencyName.name }}</span> is already provided by <span class="issue-detail">{{ issue.DuplicateDependencyName.existing_library }}</span>.</span> + + {% elif issue.InvalidMetadata %} + <span class="issue-title">Invalid Metadata</span> + <span><span class="label">Key:</span> {{ issue.InvalidMetadata.key }} | <span class="label">Issue:</span> {{ issue.InvalidMetadata.issue }}</span> + + {% elif issue.DuplicateOrdinal %} + <span class="issue-title">Duplicate Ordinal</span> + <span>Ordinal <span class="issue-detail">#{{ issue.DuplicateOrdinal.ordinal }}</span> is assigned to <span class="issue-detail">{{ issue.DuplicateOrdinal.existing_name }}</span> and <span class="issue-detail">{{ issue.DuplicateOrdinal.duplicate_name }}</span>.</span> + + {% elif issue.NoPlatform %} + <span class="issue-title">Missing Platform</span> + <span>The type library has no target platform associated with it.</span> + + {% elif issue.UnresolvedExternalReference %} + <span class="issue-title">Unresolved External Reference</span> + <span>Type <span class="issue-detail">{{ issue.UnresolvedExternalReference.name }}</span> (in <span class="issue-detail">{{ issue.UnresolvedExternalReference.container }}</span>) has no source.</span> + + {% elif issue.UnresolvedSourceReference %} + <span class="issue-title">Unresolved Source Reference</span> + <span>Type <span class="issue-detail">{{ issue.UnresolvedSourceReference.name }}</span> was not found in expected source <span class="issue-detail">{{ issue.UnresolvedSourceReference.source }}</span>.</span> + + {% elif issue.UnresolvedTypeLibrary %} + <span class="issue-title">Unresolved Type Library</span> + <span>Could not find dependency library file for <span class="issue-detail">{{ issue.UnresolvedTypeLibrary.name }}</span>.</span> + {% endif %} + +</div> +{% endfor %} + +</body> +</html>
\ No newline at end of file diff --git a/plugins/bntl_utils/src/url.rs b/plugins/bntl_utils/src/url.rs new file mode 100644 index 00000000..16fe75d2 --- /dev/null +++ b/plugins/bntl_utils/src/url.rs @@ -0,0 +1,345 @@ +use binaryninja::collaboration::{RemoteFile, RemoteFolder, RemoteProject}; +use binaryninja::rc::Ref; +use std::fmt::Display; +use std::path::PathBuf; +use thiserror::Error; +use url::Url; +use uuid::Uuid; + +#[derive(Error, Debug, PartialEq)] +pub enum BnUrlParsingError { + #[error("Invalid URL format: {0}")] + UrlParseError(#[from] url::ParseError), + + #[error("Invalid scheme: expected 'binaryninja', found '{0}'")] + InvalidScheme(String), + + #[error("Invalid Enterprise path: missing server or project GUID")] + InvalidEnterprisePath, + + #[error("Invalid server URL in enterprise path")] + InvalidServerUrl, + + #[error("Invalid UUID: {0}")] + InvalidUuid(#[from] uuid::Error), + + #[error("Unknown or unsupported URL format")] + UnknownFormat, +} + +#[derive(Error, Debug)] +pub enum BnResourceError { + #[error("Enterprise server not found for address: {0}")] + RemoteNotFound(String), + + #[error("Remote connection error: {0}")] + RemoteConnectionError(String), + + #[error("Project not found with GUID: {0}")] + ProjectNotFound(String), + + #[error("Project resource not found with GUID: {0}")] + ItemNotFound(String), + + #[error("Local filesystem error: {0}")] + IoError(#[from] std::io::Error), +} + +#[derive(Debug, Clone)] +pub enum BnResource { + RemoteProject(Ref<RemoteProject>), + RemoteProjectFile(Ref<RemoteFile>), + RemoteProjectFolder(Ref<RemoteFolder>), + /// A remote file. + RemoteFile(Url), + /// A regular file on the local filesystem. + LocalFile(PathBuf), +} + +// TODO: Make the BnUrl from this. +impl Display for BnResource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BnResource::RemoteProject(project) => write!(f, "RemoteProject({})", project.id()), + BnResource::RemoteProjectFile(file) => write!(f, "RemoteFile({})", file.id()), + BnResource::RemoteProjectFolder(folder) => write!(f, "RemoteFolder({})", folder.id()), + BnResource::RemoteFile(url) => write!(f, "RemoteFile({})", url), + BnResource::LocalFile(path) => write!(f, "LocalFile({})", path.display()), + } + } +} + +#[derive(Debug, PartialEq, Clone)] +pub enum BnParsedUrlKind { + Enterprise { + server: Url, + project_guid: Uuid, + /// Optional GUID of the project item, currently can be a folder or a file. + item_guid: Option<Uuid>, + }, + // TODO: Local projects? + RemoteFile(Url), + LocalFile(PathBuf), +} + +#[derive(Debug, Clone)] +pub struct BnParsedUrl { + pub kind: BnParsedUrlKind, + pub expression: Option<String>, +} + +impl BnParsedUrl { + pub fn parse(input: &str) -> Result<Self, BnUrlParsingError> { + let parsed = Url::parse(input)?; + if parsed.scheme() != "binaryninja" { + return Err(BnUrlParsingError::InvalidScheme( + parsed.scheme().to_string(), + )); + } + + let expression = parsed + .query_pairs() + .find(|(k, _)| k == "expr") + .map(|(_, v)| v.into_owned()); + + let kind = match parsed.host_str() { + // TODO: This should really go down the same path as the remote file parsing, if it + // TODO: matches the host of an enterprise server... But that requires us to change how + // TODO: the core outputs these enterprise URLs... + // Case: binaryninja://enterprise/... + Some("enterprise") => { + let segments: Vec<&str> = + parsed.path().split('/').filter(|s| !s.is_empty()).collect(); + + if segments.len() < 3 { + return Err(BnUrlParsingError::InvalidEnterprisePath); + } + + let (server_parts, resource_parts) = if segments.len() >= 4 { + ( + &segments[..segments.len() - 2], + &segments[segments.len() - 2..], + ) + } else { + ( + &segments[..segments.len() - 1], + &segments[segments.len() - 1..], + ) + }; + + BnParsedUrlKind::Enterprise { + server: Url::parse(&server_parts.join("/")) + .map_err(|_| BnUrlParsingError::InvalidServerUrl)?, + project_guid: Uuid::parse_str(resource_parts[0])?, + item_guid: resource_parts + .get(1) + .map(|s| Uuid::parse_str(s)) + .transpose()?, + } + } + // Case: binaryninja:///bin/ls + None | Some("") + if parsed.path().starts_with('/') && !parsed.path().starts_with("/https") => + { + BnParsedUrlKind::LocalFile(PathBuf::from(parsed.path())) + } + // Case: binaryninja:https://... + _ => { + let path = parsed.path(); + if path.starts_with("https:/") || path.starts_with("http:/") { + let nested_url = path.replacen(":/", "://", 1); + BnParsedUrlKind::RemoteFile( + Url::parse(&nested_url).map_err(BnUrlParsingError::UrlParseError)?, + ) + } else { + return Err(BnUrlParsingError::UnknownFormat); + } + } + }; + + Ok(BnParsedUrl { kind, expression }) + } + + pub fn to_resource(&self) -> Result<BnResource, BnResourceError> { + match &self.kind { + BnParsedUrlKind::Enterprise { + server, + project_guid, + item_guid, + } => { + // NOTE: We must strip the trailing slash from the server URL, because the core will + // not accept it otherwise, we should probably have a fuzzy get_remote_by_address here, + // so we can accept either with or without the trailing slash, but for now we'll just + // strip it. + let server_addr = server.as_str().strip_suffix('/').unwrap_or(server.as_str()); + let remote = binaryninja::collaboration::get_remote_by_address(server_addr) + .ok_or_else(|| BnResourceError::RemoteNotFound(server_addr.to_string()))?; + if !remote.is_connected() { + remote.connect().map_err(|_| { + BnResourceError::RemoteConnectionError(server_addr.to_string()) + })?; + } + + let project = remote + .get_project_by_id(&project_guid.to_string()) + .ok() + .flatten() + .ok_or_else(|| BnResourceError::ProjectNotFound(project_guid.to_string()))?; + + match item_guid { + Some(item_guid) => { + let item_guid_str = item_guid.to_string(); + + // Check if it's a folder first + if let Some(folder) = + project.get_folder_by_id(&item_guid_str).ok().flatten() + { + return Ok(BnResource::RemoteProjectFolder(folder)); + } + + // Then check if it's a file + let file = project + .get_file_by_id(&item_guid_str) + .ok() + .flatten() + .ok_or_else(|| BnResourceError::ItemNotFound(item_guid_str))?; + + Ok(BnResource::RemoteProjectFile(file)) + } + None => Ok(BnResource::RemoteProject(project)), + } + } + BnParsedUrlKind::RemoteFile(remote_url) => { + Ok(BnResource::RemoteFile(remote_url.clone())) + } + BnParsedUrlKind::LocalFile(local_path) => Ok(BnResource::LocalFile(local_path.clone())), + } + } +} + +impl Display for BnParsedUrl { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.kind { + BnParsedUrlKind::Enterprise { + server, + project_guid, + item_guid, + } => write!( + f, + "binaryninja://enterprise/{}/{}{}", + server.as_str().strip_suffix('/').unwrap_or(server.as_str()), + project_guid, + item_guid + .map(|guid| format!("/{}", guid)) + .unwrap_or_default() + ), + BnParsedUrlKind::RemoteFile(remote_url) => write!(f, "{}", remote_url), + BnParsedUrlKind::LocalFile(local_path) => { + write!(f, "binaryninja:///{}", local_path.display()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_parse_enterprise_full() { + let input = "binaryninja://enterprise/https://enterprise.test.com/0268b954-0d7b-41c3-a603-960a59fdd0f7/0268b954-0d7b-41c3-a603-960a59fdd0f6?expr=sub_1234"; + let action = BnParsedUrl::parse(input).unwrap(); + + if let BnParsedUrlKind::Enterprise { + server, + project_guid, + item_guid: project_file, + } = action.kind + { + assert_eq!(server.as_str(), "https://enterprise.test.com/"); + assert_eq!( + project_guid, + Uuid::parse_str("0268b954-0d7b-41c3-a603-960a59fdd0f7").unwrap() + ); + assert_eq!( + project_file, + Some(Uuid::parse_str("0268b954-0d7b-41c3-a603-960a59fdd0f6").unwrap()) + ); + } else { + panic!("Wrong target type"); + } + assert_eq!(action.expression, Some("sub_1234".to_string())); + } + + #[test] + fn test_parse_enterprise_no_file() { + let input = "binaryninja://enterprise/https://enterprise.test.com/0268b954-0d7b-41c3-a603-960a59fdd0f7/"; + let action = BnParsedUrl::parse(input).unwrap(); + + if let BnParsedUrlKind::Enterprise { + project_guid, + item_guid: project_file, + .. + } = action.kind + { + assert_eq!( + project_guid, + Uuid::parse_str("0268b954-0d7b-41c3-a603-960a59fdd0f7").unwrap() + ); + assert_eq!(project_file, None); + } else { + panic!("Wrong target type"); + } + } + + #[test] + fn test_parse_remote_file() { + let input = "binaryninja:https://captf.com/2015/plaidctf/pwnable/datastore.elf?expr=main"; + let action = BnParsedUrl::parse(input).unwrap(); + + match action.kind { + BnParsedUrlKind::RemoteFile(url) => { + assert_eq!(url.host_str(), Some("captf.com")); + assert!(url.path().ends_with("datastore.elf")); + } + _ => panic!("Expected RemoteFile"), + } + assert_eq!(action.expression, Some("main".to_string())); + } + + #[test] + fn test_parse_local_file() { + let input = "binaryninja:///bin/ls?expr=sub_2830"; + let action = BnParsedUrl::parse(input).unwrap(); + + match action.kind { + BnParsedUrlKind::LocalFile(path) => assert_eq!(path.to_string_lossy(), "/bin/ls"), + _ => panic!("Expected LocalFile"), + } + assert_eq!(action.expression, Some("sub_2830".to_string())); + } + + #[test] + fn test_invalid_scheme() { + let input = "https://google.com"; + let result = BnParsedUrl::parse(input); + assert!(matches!(result, Err(BnUrlParsingError::InvalidScheme(_)))); + } + + #[test] + fn test_missing_enterprise_guid() { + let input = "binaryninja://enterprise/https://internal.us/"; + let result = BnParsedUrl::parse(input); + assert_eq!( + result.unwrap_err(), + BnUrlParsingError::InvalidEnterprisePath + ); + } + + #[test] + fn test_invalid_uuid_format() { + let input = "binaryninja://enterprise/https://internal.us/not-a-uuid/"; + let result = BnParsedUrl::parse(input); + assert!(matches!(result, Err(BnUrlParsingError::InvalidUuid(_)))); + } +} diff --git a/plugins/bntl_utils/src/validate.rs b/plugins/bntl_utils/src/validate.rs new file mode 100644 index 00000000..ecd0978d --- /dev/null +++ b/plugins/bntl_utils/src/validate.rs @@ -0,0 +1,347 @@ +use crate::schema::BntlSchema; +use binaryninja::platform::Platform; +use binaryninja::qualified_name::QualifiedName; +use binaryninja::rc::Ref; +use binaryninja::types::TypeLibrary; +use minijinja::{context, Environment}; +use serde::Serialize; +use std::collections::{HashMap, HashSet}; +use std::env::temp_dir; +use std::fmt::Display; + +#[derive(Debug, PartialEq, PartialOrd, Clone, Eq, Hash, Serialize)] +pub enum ValidateIssue { + DuplicateGUID { + guid: String, + existing_library: String, + }, + DuplicateDependencyName { + name: String, + existing_library: String, + }, + InvalidMetadata { + key: String, + issue: String, + }, + DuplicateOrdinal { + ordinal: u64, + existing_name: String, + duplicate_name: String, + }, + NoPlatform, + UnresolvedExternalReference { + name: String, + container: String, + }, + UnresolvedSourceReference { + name: String, + source: String, + }, + UnresolvedTypeLibrary { + name: String, + }, // TODO: Overlapping type name of platform? + // TODO: E.g. a type is found in the type library, and also in the platform. +} + +impl Display for ValidateIssue { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ValidateIssue::DuplicateGUID { + guid, + existing_library, + } => { + write!( + f, + "Duplicate GUID: '{}' is already used by library '{}'", + guid, existing_library + ) + } + ValidateIssue::DuplicateDependencyName { + name, + existing_library, + } => { + write!( + f, + "Duplicate Dependency Name: '{}' is already provided by '{}'", + name, existing_library + ) + } + ValidateIssue::InvalidMetadata { key, issue } => { + write!(f, "Invalid Metadata: Key '{}' - {}", key, issue) + } + ValidateIssue::DuplicateOrdinal { + ordinal, + existing_name, + duplicate_name, + } => { + write!( + f, + "Duplicate Ordinal: #{} is assigned to both '{}' and '{}'", + ordinal, existing_name, duplicate_name + ) + } + ValidateIssue::NoPlatform => { + write!( + f, + "Missing Platform: The type library has no target platform associated with it" + ) + } + ValidateIssue::UnresolvedExternalReference { name, container } => { + write!( + f, + "Unresolved External Reference: Type '{}' referenced inside '{}' is marked as external but has no source", + name, container + ) + } + ValidateIssue::UnresolvedSourceReference { name, source } => { + write!( + f, + "Unresolved Source Reference: Type '{}' expects source '{}', but it wasn't found there", + name, source + ) + } + ValidateIssue::UnresolvedTypeLibrary { name } => { + write!( + f, + "Unresolved Type Library: Could not find dependency library file for '{}'", + name + ) + } + } + } +} + +#[derive(Debug, Default)] +pub struct ValidateResult { + pub issues: Vec<ValidateIssue>, +} + +impl ValidateResult { + /// Render the validation report as HTML. + pub fn render_report(&self) -> Result<String, minijinja::Error> { + let mut environment = Environment::new(); + // Remove trailing lines for blocks, this is required for Markdown tables. + environment.set_trim_blocks(true); + minijinja_embed::load_templates!(&mut environment); + let tmpl = environment.get_template("validate.html")?; + tmpl.render(context!(issues => self.issues)) + } +} + +#[derive(Debug, Default, Clone)] +pub struct TypeLibValidater { + pub seen_guids: HashMap<String, String>, + // TODO: This needs to be by platform as well. + pub seen_dependency_names: HashMap<String, String>, + /// These are the type libraries that are accessible to the type library under validation. + /// + /// Used to validate external references. + pub type_libraries: Vec<Ref<TypeLibrary>>, + /// Built from the available type libraries. + pub valid_external_references: HashSet<QualifiedName>, +} + +impl TypeLibValidater { + pub fn new() -> Self { + Self { + seen_guids: HashMap::new(), + seen_dependency_names: HashMap::new(), + type_libraries: Vec::new(), + valid_external_references: HashSet::new(), + } + } + + /// These are the type libraries that are accessible to the type library under validation. + /// + /// Used to validate external references. + pub fn with_type_libraries(mut self, type_libraries: Vec<Ref<TypeLibrary>>) -> Self { + self.type_libraries = type_libraries; + for type_lib in &self.type_libraries { + for ty in &type_lib.named_types() { + self.valid_external_references.insert(ty.name); + } + for obj in &type_lib.named_objects() { + self.valid_external_references.insert(obj.name); + } + } + self + } + + /// The platform that is accessible to the type library under validation. + /// + /// Used to validate external references. + pub fn with_platform(mut self, platform: &Platform) -> Self { + for ty in &platform.types() { + self.valid_external_references.insert(ty.name); + } + self + } + + pub fn validate(&mut self, type_lib: &TypeLibrary) -> ValidateResult { + let mut result = ValidateResult::default(); + + if type_lib.platform_names().is_empty() { + result.issues.push(ValidateIssue::NoPlatform); + } + + if let Some(issue) = self.validate_guid(type_lib) { + result.issues.push(issue); + } + + if let Some(issue) = self.validate_dependency_name(type_lib) { + result.issues.push(issue); + } + + result.issues.extend(self.validate_ordinals(type_lib)); + result + .issues + .extend(self.validate_external_references(type_lib)); + + // TODO: This is currently disabled because it's too slow. + // result.issues.extend(self.validate_source_files(type_lib)); + + result + } + + pub fn validate_guid(&mut self, type_lib: &TypeLibrary) -> Option<ValidateIssue> { + match self.seen_guids.insert(type_lib.guid(), type_lib.name()) { + None => None, + Some(existing_library) => Some(ValidateIssue::DuplicateGUID { + guid: type_lib.guid(), + existing_library, + }), + } + } + + pub fn validate_dependency_name(&mut self, type_lib: &TypeLibrary) -> Option<ValidateIssue> { + match self + .seen_dependency_names + .insert(type_lib.dependency_name(), type_lib.name()) + { + None => None, + Some(existing_library) => Some(ValidateIssue::DuplicateDependencyName { + name: type_lib.dependency_name(), + existing_library, + }), + } + } + + pub fn validate_source_files(&self, type_lib: &TypeLibrary) -> Vec<ValidateIssue> { + let mut issues = Vec::new(); + let tmp_type_lib_path = temp_dir().join(type_lib.name()); + if !type_lib.decompress_to_file(&tmp_type_lib_path) { + tracing::error!( + "Failed to decompress type library to temporary file: {}", + type_lib.name() + ); + return issues; + } + let schema = BntlSchema::from_path(&tmp_type_lib_path); + for (src, types) in schema.to_source_map() { + let Some(dep_type_lib) = self.type_libraries.iter().find(|tl| tl.name() == src) else { + issues.push(ValidateIssue::UnresolvedTypeLibrary { + name: src.to_string(), + }); + continue; + }; + + for ty in &types { + let qualified_name = QualifiedName::from(ty); + let is_named_ty = dep_type_lib + .get_named_type(qualified_name.clone()) + .is_none(); + let is_named_obj = dep_type_lib.get_named_object(qualified_name).is_none(); + if !is_named_ty && !is_named_obj { + issues.push(ValidateIssue::UnresolvedSourceReference { + name: ty.to_string(), + source: src.to_string(), + }); + } + } + } + issues + } + + pub fn validate_external_references(&self, type_lib: &TypeLibrary) -> Vec<ValidateIssue> { + let mut issues = Vec::new(); + for ty in &type_lib.named_types() { + crate::helper::visit_type_reference(&ty.ty, &mut |ntr| { + if !self.valid_external_references.contains(&ntr.name()) { + issues.push(ValidateIssue::UnresolvedExternalReference { + name: ntr.name().to_string(), + container: ty.name.to_string(), + }); + } + }) + } + for obj in &type_lib.named_objects() { + crate::helper::visit_type_reference(&obj.ty, &mut |ntr| { + if !self.valid_external_references.contains(&ntr.name()) { + issues.push(ValidateIssue::UnresolvedExternalReference { + name: ntr.name().to_string(), + container: obj.name.to_string(), + }); + } + }) + } + issues + } + + pub fn validate_ordinals(&self, type_lib: &TypeLibrary) -> Vec<ValidateIssue> { + let Some(metadata_key_md) = type_lib.query_metadata("metadata") else { + return vec![]; + }; + let Some(metadata_key_str) = metadata_key_md.get_string() else { + return vec![ValidateIssue::InvalidMetadata { + key: "metadata".to_owned(), + issue: "Expected string".to_owned(), + }]; + }; + + let Some(metadata_map_md) = type_lib.query_metadata(&metadata_key_str.to_string_lossy()) + else { + return vec![ValidateIssue::InvalidMetadata { + key: metadata_key_str.to_string_lossy().to_string(), + issue: "Missing metadata map key".to_owned(), + }]; + }; + + let Some(metadata_map) = metadata_map_md.get_value_store() else { + return vec![ValidateIssue::InvalidMetadata { + key: metadata_key_str.to_string_lossy().to_string(), + issue: "Expected value store".to_owned(), + }]; + }; + + let mut discovered_ordinals = HashMap::new(); + let mut issues = Vec::new(); + for (key, value) in metadata_map.iter() { + let Ok(ordinal_num) = key.parse::<u64>() else { + issues.push(ValidateIssue::InvalidMetadata { + key: key.to_string(), + issue: "Expected ordinal number".to_owned(), + }); + continue; + }; + + let Some(value_bn_str) = value.get_string() else { + issues.push(ValidateIssue::InvalidMetadata { + key: key.to_string(), + issue: "Expected string".to_owned(), + }); + continue; + }; + let value_str = value_bn_str.to_string_lossy().to_string(); + + match discovered_ordinals.insert(ordinal_num, value_str.clone()) { + None => (), + Some(existing_ordinal) => issues.push(ValidateIssue::DuplicateOrdinal { + ordinal: ordinal_num, + existing_name: existing_ordinal, + duplicate_name: value_str, + }), + } + } + issues + } +} diff --git a/plugins/bntl_utils/src/winmd.rs b/plugins/bntl_utils/src/winmd.rs new file mode 100644 index 00000000..1cf99ab2 --- /dev/null +++ b/plugins/bntl_utils/src/winmd.rs @@ -0,0 +1,592 @@ +//! Import windows metadata types into a Binary Ninja type library. + +use std::collections::HashMap; +use std::num::NonZeroUsize; +use std::path::PathBuf; +use thiserror::Error; + +use binaryninja::architecture::Architecture; +use binaryninja::platform::Platform; +use binaryninja::qualified_name::QualifiedName; +use binaryninja::rc::Ref; +use binaryninja::types::{ + EnumerationBuilder, FunctionParameter, MemberAccess, MemberScope, NamedTypeReference, + NamedTypeReferenceClass, StructureBuilder, StructureType, Type, TypeBuilder, TypeLibrary, +}; + +use info::{LibraryName, MetadataFunctionInfo, MetadataInfo, MetadataTypeInfo, MetadataTypeKind}; + +pub mod info; +pub mod translate; + +#[derive(Error, Debug)] +pub enum ImportError { + #[error("no files were provided")] + NoFiles, + #[error("the type name '{0}' is not handled")] + UnhandledType(String), + #[error("failed to translate windows metadata")] + TransactionError(#[from] translate::TranslationError), + #[error("the type '{0}' has an unhandled size")] + UnhandledTypeSize(&'static str), +} + +#[derive(Debug)] +pub struct WindowsMetadataImporter { + info: MetadataInfo, + // TODO: If we can replace / add this with type libraries we can make multi-pass importer. + type_lookup: HashMap<(String, String), MetadataTypeInfo>, + address_size: usize, + integer_size: usize, +} + +impl WindowsMetadataImporter { + pub fn new() -> Self { + Self { + info: MetadataInfo::default(), + type_lookup: HashMap::new(), + address_size: 8, + integer_size: 8, + } + } + + #[allow(dead_code)] + pub fn new_with_info(info: MetadataInfo) -> Self { + let mut res = Self::new(); + res.info = info; + res.build_type_lookup(); + res + } + + pub fn with_files(mut self, paths: &[PathBuf]) -> Result<Self, ImportError> { + let mut files = Vec::new(); + for path in paths { + let file = windows_metadata::reader::File::read(path).expect("Failed to read file"); + files.push(file); + } + self.info = translate::WindowsMetadataTranslator::new().translate(files)?; + // We updated info, so we must rebuild the lookup table. + self.build_type_lookup(); + Ok(self) + } + + pub fn with_platform(mut self, platform: &Platform) -> Self { + // TODO: platform.address_size() + self.address_size = platform.arch().address_size(); + self.integer_size = platform.arch().default_integer_size(); + self + } + + /// Build the lookup table for us to use when referencing types. + /// + /// Should be called anytime we update `self.info`. + fn build_type_lookup(&mut self) { + for ty in &self.info.types { + if let Some(_existing) = self + .type_lookup + .insert((ty.namespace.clone(), ty.name.clone()), ty.clone()) + { + tracing::warn!( + "Duplicate type name '{}' found when building type lookup", + ty.name + ); + } + } + } + + pub fn import(&self, platform: &Platform) -> Result<Vec<Ref<TypeLibrary>>, ImportError> { + // TODO: We need to take all of these enums and figure out where to put them. + let mut test = self.info.clone(); + let constant_enums = test.create_constant_enums(); + // TODO: Creating zero width enums + test.types.extend(constant_enums); + let partitioned_info = test.partitioned(); + + let mut type_libs = Vec::new(); + for (name, info) in partitioned_info.libraries { + let type_lib_name = match name { + LibraryName::Module(module_name) => module_name.clone(), + LibraryName::Namespace(ns_name) => { + // TODO: We might need to do something different for namespaced type libraries in the future. + ns_name.clone() + } + }; + let til = TypeLibrary::new(platform.arch(), &type_lib_name); + til.add_platform(platform); + til.set_dependency_name(&type_lib_name); + for ty in &info.metadata.types { + self.import_type(&til, &ty)?; + } + for func in &info.metadata.functions { + self.import_function(&til, &func)?; + } + for (name, library_name) in &info.external_references { + let qualified_name = QualifiedName::from(name.clone()); + match library_name { + LibraryName::Namespace(source) => { + // TODO: We might need to do something different for namespaced type libraries in the future. + til.add_type_source(qualified_name, source); + } + LibraryName::Module(source) => { + til.add_type_source(qualified_name, source); + } + } + } + + type_libs.push(til); + } + + Ok(type_libs) + } + + pub fn import_function( + &self, + til: &TypeLibrary, + func: &MetadataFunctionInfo, + ) -> Result<(), ImportError> { + // TODO: Handle ordinals? Ordinals exist in binaries that need to be parsed, maybe we + // TODO: make another handler for that + let qualified_name = QualifiedName::from(func.name.clone()); + let ty = self.convert_type_kind(&func.ty)?; + til.add_named_object(qualified_name, &ty); + Ok(()) + } + + pub fn import_type( + &self, + til: &TypeLibrary, + type_info: &MetadataTypeInfo, + ) -> Result<(), ImportError> { + let qualified_name = QualifiedName::from(type_info.name.clone()); + let ty = self.convert_type_kind(&type_info.kind)?; + til.add_named_type(qualified_name, &ty); + Ok(()) + } + + pub fn convert_type_kind(&self, kind: &MetadataTypeKind) -> Result<Ref<Type>, ImportError> { + match kind { + MetadataTypeKind::Void => Ok(Type::void()), + MetadataTypeKind::Bool { size: None } => Ok(Type::bool()), + MetadataTypeKind::Bool { size: Some(size) } => { + Ok(TypeBuilder::bool().set_width(*size).finalize()) + } + MetadataTypeKind::Integer { size, is_signed } => { + Ok(Type::int(size.unwrap_or(self.integer_size), *is_signed)) + } + MetadataTypeKind::Character { size: 1 } => Ok(Type::int(1, true)), + MetadataTypeKind::Character { size } => Ok(Type::wide_char(*size)), + MetadataTypeKind::Float { size } => Ok(Type::float(*size)), + MetadataTypeKind::Pointer { + is_const, + is_pointee_const: _is_pointee_const, + target, + } => { + let target_ty = self.convert_type_kind(target)?; + Ok(Type::pointer_of_width( + &target_ty, + self.address_size, + *is_const, + false, + None, + )) + } + MetadataTypeKind::Array { element, count } => { + let element_ty = self.convert_type_kind(element)?; + Ok(Type::array(&element_ty, *count as u64)) + } + MetadataTypeKind::Struct { fields, is_packed } => { + let mut structure = StructureBuilder::new(); + // Current offset in bytes + let mut current_byte_offset = 0usize; + + // TODO: Change how this operates now that we have an is_packed flag. + // Used to add tail padding to satisfy alignment requirements. + let mut max_alignment = 0usize; + // We need to look ahead to figure out when bitfields end and adjust current_byte_offset accordingly. + let mut field_iter = fields.iter().peekable(); + while let Some(field) = field_iter.next() { + let field_ty = self.convert_type_kind(&field.ty)?; + let field_size = self.type_kind_size(&field.ty)?; + let field_alignment = self.type_kind_alignment(&field.ty)?; + max_alignment = max_alignment.max(field_alignment); + if let Some((bit_pos, bit_width)) = field.bitfield { + let current_bit_offset = current_byte_offset * 8; + let field_bit_offset = current_bit_offset + bit_pos as usize; + // TODO: member access and member scope have definitions inside winmd we can use. + structure.insert_bitwise( + &field_ty, + &field.name, + field_bit_offset as u64, + Some(bit_width), + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + + if let Some(next_field) = field_iter.peek() { + if next_field.bitfield.is_some() { + // Continue as if we are in the same storage unit (no alignment) + current_byte_offset = (current_bit_offset + bit_width as usize) / 8; + } else { + // Find the start of the storage unit. + // if we are at byte 1 of u32 (align 4), storage starts at 0. + // if we are at byte 5 of u32 (align 4), storage starts at 4. + let storage_start = + (current_byte_offset / field_alignment) * field_alignment; + // Jump to the end of that storage unit. + current_byte_offset = storage_start + field_size; + } + } + } else { + // Align the field placement based on the current field alignment. + let aligned_current_offset = + align_up(current_byte_offset as u64, field_alignment as u64); + structure.insert( + &field_ty, + &field.name, + aligned_current_offset, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + current_byte_offset = aligned_current_offset as usize + field_size; + } + } + structure.alignment(max_alignment); + + // TODO: Only add tail padding if we are not packed? I think we still need to do more. + if *is_packed { + structure.packed(true); + } else { + let total_size = align_up(current_byte_offset as u64, max_alignment as u64); + structure.width(total_size); + } + + Ok(Type::structure(&structure.finalize())) + } + MetadataTypeKind::Enum { ty, variants } => { + // NOTE: A void type may be returned by synthetic constant enums, which is why we + // do not error when there is a zero width enum. + let enum_ty = self.convert_type_kind(ty)?; + let mut builder = EnumerationBuilder::new(); + for (name, value) in variants { + builder.insert(name, *value); + } + Ok(Type::enumeration( + &builder.finalize(), + NonZeroUsize::new(enum_ty.width() as usize) + .unwrap_or(NonZeroUsize::new(self.integer_size).unwrap()), + enum_ty.is_signed().contents, + )) + } + MetadataTypeKind::Function { + params, + return_type, + is_vararg, + } => { + let return_ty = self.convert_type_kind(return_type)?; + let mut bn_params = Vec::new(); + for param in params { + let param_ty = self.convert_type_kind(¶m.ty)?; + bn_params.push(FunctionParameter::new(param_ty, param.name.clone(), None)); + } + Ok(Type::function(&return_ty, bn_params, *is_vararg)) + } + MetadataTypeKind::Reference { name, namespace } => { + // We are required to set the ID here since type libraries seem to only look up through + // the ID, and never fall back to name lookup. This is strange considering you must also + // set the types source to the given library, which seems counterintuitive. + // TODO: Add kind to ntr. + let ntr = NamedTypeReference::new_with_id( + NamedTypeReferenceClass::TypedefNamedTypeClass, + &format!("{}::{}", namespace, name), + name, + ); + // TODO: Type alignment? + let type_size = self.type_kind_size(kind)?; + Ok(TypeBuilder::named_type(&ntr) + .set_width(type_size) + .set_alignment(type_size) + .finalize()) + } + MetadataTypeKind::Union { fields } => { + let mut union = StructureBuilder::new(); + union.structure_type(StructureType::UnionStructureType); + + let mut max_alignment = 0usize; + // We need to look ahead to figure out when bitfields end and adjust current_byte_offset accordingly. + let mut field_iter = fields.iter().peekable(); + while let Some(field) = field_iter.next() { + let field_ty = self.convert_type_kind(&field.ty)?; + let field_alignment = self.type_kind_alignment(&field.ty)?; + max_alignment = max_alignment.max(field_alignment); + union.insert( + &field_ty, + &field.name, + 0, + false, + MemberAccess::PublicAccess, + MemberScope::NoScope, + ); + } + + union.alignment(max_alignment); + Ok(Type::structure(&union.finalize())) + } + } + } + + /// Retrieve the size of a type kind in bytes, references to types will be looked up + /// such that we can determine the size of structures with references as fields. + pub fn type_kind_size(&self, kind: &MetadataTypeKind) -> Result<usize, ImportError> { + match kind { + MetadataTypeKind::Void => Ok(0), + MetadataTypeKind::Bool { size } => Ok(size.unwrap_or(self.integer_size)), + MetadataTypeKind::Integer { size, .. } => Ok(size.unwrap_or(self.integer_size)), + MetadataTypeKind::Character { size } => Ok(*size), + MetadataTypeKind::Float { size } => Ok(*size), + MetadataTypeKind::Pointer { .. } => Ok(self.address_size), + MetadataTypeKind::Array { element, count } => { + let elem_size = self.type_kind_size(element)?; + Ok(elem_size * *count) + } + MetadataTypeKind::Struct { fields, is_packed } => { + let mut current_offset = 0usize; + let mut max_struct_alignment = 1usize; + for field in fields { + let field_size = self.type_kind_size(&field.ty)?; + let field_alignment = if *is_packed { + 1 + } else { + self.type_kind_alignment(&field.ty)? + }; + max_struct_alignment = max_struct_alignment.max(field_alignment); + current_offset = + align_up(current_offset as u64, field_alignment as u64) as usize; + current_offset += field_size; + } + // Tail padding is only needed if not packed. + let final_alignment = if *is_packed { 1 } else { max_struct_alignment }; + let total_size = align_up(current_offset as u64, final_alignment as u64) as usize; + Ok(total_size) + } + MetadataTypeKind::Union { fields } => { + let mut largest_field_size = 0usize; + for field in fields { + let field_size = self.type_kind_size(&field.ty)?; + largest_field_size = largest_field_size.max(field_size); + } + Ok(largest_field_size) + } + MetadataTypeKind::Enum { ty, .. } => self.type_kind_size(ty), + MetadataTypeKind::Function { .. } => Err(ImportError::UnhandledTypeSize( + "Function types are not sized", + )), + MetadataTypeKind::Reference { name, namespace } => { + // Look up the type and return its size. + let Some(ty_info) = self.type_lookup.get(&(namespace.clone(), name.clone())) else { + // This should really only happen if we did not specify all the required winmd files. + // tracing::error!( + // "Failed to find type '{}' when looking up type size for reference", + // name + // ); + return Ok(1); + }; + self.type_kind_size(&ty_info.kind) + } + } + } + + pub fn type_kind_alignment(&self, kind: &MetadataTypeKind) -> Result<usize, ImportError> { + match kind { + MetadataTypeKind::Bool { size: None } => Ok(1), + MetadataTypeKind::Bool { size } => Ok(size.unwrap_or(self.integer_size)), + // TODO: Clean this stuff up. + MetadataTypeKind::Character { size } => Ok(*size), + MetadataTypeKind::Integer { size: Some(1), .. } => Ok(1), + MetadataTypeKind::Integer { size: Some(2), .. } => Ok(2), + MetadataTypeKind::Integer { size: Some(4), .. } => Ok(4), + MetadataTypeKind::Integer { size: Some(8), .. } + | MetadataTypeKind::Float { size: 8 } + | MetadataTypeKind::Pointer { .. } => Ok(self.address_size), // 8 on x64 + MetadataTypeKind::Array { element, .. } => self.type_kind_alignment(element), + MetadataTypeKind::Struct { fields, is_packed } => { + if *is_packed { + return Ok(1); + } + let mut max_align = 1usize; + for field in fields { + max_align = max_align.max(self.type_kind_alignment(&field.ty)?); + } + Ok(max_align) + } + MetadataTypeKind::Union { fields } => { + let mut max_align = 1usize; + for field in fields { + max_align = max_align.max(self.type_kind_alignment(&field.ty)?); + } + Ok(max_align) + } + MetadataTypeKind::Reference { name, namespace } => { + let Some(ty_info) = self.type_lookup.get(&(namespace.clone(), name.clone())) else { + // TODO: Failed to find it in local type lookup, try type libraries? + // tracing::error!( + // "Failed to find type '{}' when looking up type alignment for reference", + // name + // ); + return Ok(4); + }; + self.type_kind_alignment(&ty_info.kind) + } + _ => Ok(4), + } + } +} + +// Aligns an offset up to the nearest multiple of `align`. +fn align_up(offset: u64, align: u64) -> u64 { + if align == 0 { + return offset; + } + let mask = align - 1; + (offset + mask) & !mask +} + +#[cfg(test)] +mod tests { + use super::info::{ + MetadataFieldInfo, MetadataImportInfo, MetadataImportMethod, MetadataModuleInfo, + }; + use super::*; + use binaryninja::architecture::CoreArchitecture; + use binaryninja::types::TypeClass; + + #[test] + fn test_import_type() { + // We must initialize binary ninja to access architectures. + let _session = binaryninja::headless::Session::new().expect("Failed to create session"); + + let mut info = MetadataInfo::default(); + info.functions = vec![MetadataFunctionInfo { + name: "MyFunction".to_string(), + ty: MetadataTypeKind::Function { + params: vec![], + return_type: Box::new(MetadataTypeKind::Void), + is_vararg: false, + }, + namespace: "Win32.Test".to_string(), + import_info: Some(MetadataImportInfo { + method: MetadataImportMethod::ByName("MyFunction".to_string()), + module: MetadataModuleInfo { + name: "TestModule.dll".to_string(), + }, + }), + }]; + info.types = vec![ + MetadataTypeInfo { + name: "Bar".to_string(), + kind: MetadataTypeKind::Integer { + size: Some(4), + is_signed: true, + }, + namespace: "Win32.Test".to_string(), + }, + MetadataTypeInfo { + name: "TestType".to_string(), + kind: MetadataTypeKind::Struct { + fields: vec![ + MetadataFieldInfo { + name: "field1".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(4), + is_signed: false, + }, + is_const: false, + bitfield: None, + }, + // TODO: Add more fields to verify bitfields, and const fields. + MetadataFieldInfo { + name: "field2_0".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(4), + is_signed: true, + }, + is_const: true, + bitfield: Some((0, 1)), + }, + MetadataFieldInfo { + name: "field2_1".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(4), + is_signed: true, + }, + is_const: true, + bitfield: Some((1, 1)), + }, + MetadataFieldInfo { + name: "field3".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(2), + is_signed: true, + }, + is_const: true, + bitfield: None, + }, + MetadataFieldInfo { + name: "field4".to_string(), + ty: MetadataTypeKind::Pointer { + is_pointee_const: false, + is_const: false, + target: Box::new(MetadataTypeKind::Reference { + namespace: "Win32.Test".to_string(), + name: "Bar".to_string(), + }), + }, + is_const: false, + bitfield: None, + }, + ], + is_packed: false, + }, + namespace: "Foo".to_string(), + }, + ]; + let importer = WindowsMetadataImporter::new_with_info(info); + let x86 = CoreArchitecture::by_name("x86").expect("No x86 architecture"); + let platform = Platform::by_name("windows-x86").expect("No windows-x86 platform"); + let type_libraries = importer.import(&platform).expect("Failed to import types"); + assert_eq!(type_libraries.len(), 1); + let til = type_libraries.first().expect("No type libraries"); + assert_eq!(til.named_types().len(), 1); + let first_ty = til + .named_types() + .iter() + .next() + .expect("No types in library"); + assert_eq!(first_ty.name.to_string(), "TestType"); + assert_eq!(first_ty.ty.type_class(), TypeClass::StructureTypeClass); + let first_ty_struct = first_ty + .ty + .get_structure() + .expect("Type is not a structure"); + assert_eq!(first_ty_struct.members().len(), 5); + let mut structure_fields = first_ty_struct.members().iter(); + + for member in first_ty_struct.members() { + println!(" +{}: {}", member.offset, member.name.to_string()) + } + + // TODO: Finish this! + assert!(false); + // let first_member = structure_fields.next().expect("No fields in structure"); + // assert_eq!(first_member.name.to_string(), "field1"); + // assert_eq!(first_member.ty, TypeClass::IntegerTypeClass); + // let second_member = structure_fields.next().expect("No fields in structure"); + // assert_eq!(second_member.name.to_string(), "field2_0"); + // assert_eq!(second_member.type.type_class(), TypeClass::IntegerTypeClass); + // let third_member = structure_fields.next().expect("No fields in structure"); + // assert_eq!(third_member.name.to_string(), "field2_1"); + // assert_eq!(third_member.type.type_class(), TypeClass::IntegerTypeClass); + // let fourth_member = structure_fields.next().expect("No fields in structure"); + } +} diff --git a/plugins/bntl_utils/src/winmd/info.rs b/plugins/bntl_utils/src/winmd/info.rs new file mode 100644 index 00000000..4db30c7a --- /dev/null +++ b/plugins/bntl_utils/src/winmd/info.rs @@ -0,0 +1,430 @@ +//! Metadata information extracted from Windows metadata files. +//! +//! While we could use the direct representation, this is easier to work with. + +use std::collections::{HashMap, HashSet}; + +#[derive(Debug, Default, Clone)] +pub struct MetadataInfo { + pub types: Vec<MetadataTypeInfo>, + pub functions: Vec<MetadataFunctionInfo>, + pub constants: Vec<MetadataConstantInfo>, +} + +impl MetadataInfo { + /// Partitions the metadata into a map of libraries, where each library contains types and functions + /// that belong to that library. This is used when mapping metadata info to type libraries. + pub fn partitioned(&self) -> PartitionedMetadataInfo { + let mut result_map: HashMap<LibraryName, LibraryInfo> = HashMap::new(); + + // Map of namespace to module names that use it. + let mut namespace_dependencies: HashMap<String, HashSet<String>> = HashMap::new(); + for func in &self.functions { + if let Some(import) = &func.import_info { + namespace_dependencies + .entry(func.namespace.clone()) + .or_default() + .insert(import.module.name.clone()); + } + } + + let namespace_to_library_name = |ns: &str| -> LibraryName { + match namespace_dependencies.get(ns) { + Some(modules) if modules.len() == 1 => { + LibraryName::Module(modules.iter().next().unwrap().clone()) + } + _ => LibraryName::Namespace(ns.to_string()), + } + }; + + for func in &self.functions { + let dest_lib = match &func.import_info { + Some(info) => LibraryName::Module(info.module.name.clone()), + None => LibraryName::Namespace(func.namespace.clone()), + }; + let entry = result_map.entry(dest_lib.clone()).or_default(); + func.ty.visit_references(&mut |ns, name| { + let library_name = namespace_to_library_name(ns); + if dest_lib != library_name { + entry + .external_references + .insert(name.to_string(), library_name); + } + }); + entry.metadata.functions.push(func.clone()); + } + + for ty in &self.types { + let dest_lib = namespace_to_library_name(&ty.namespace); + let entry = result_map.entry(dest_lib.clone()).or_default(); + ty.kind.visit_references(&mut |ns, name| { + let library_name = namespace_to_library_name(ns); + if dest_lib != library_name { + entry + .external_references + .insert(name.to_string(), library_name); + } + }); + entry.metadata.types.push(ty.clone()); + } + + for constant in &self.constants { + let dest_lib = namespace_to_library_name(&constant.namespace); + let entry = result_map.entry(dest_lib.clone()).or_default(); + constant.ty.visit_references(&mut |ns, name| { + let library_name = namespace_to_library_name(ns); + if dest_lib != library_name { + entry + .external_references + .insert(name.to_string(), library_name); + } + }); + entry.metadata.constants.push(constant.clone()); + } + + PartitionedMetadataInfo { + libraries: result_map, + } + } + + pub fn create_constant_enums(&self) -> Vec<MetadataTypeInfo> { + // Group constants by their type, if there are multiple constants with the same type, we + // will make an enum out of them, once that is done, we will take overlapping constants + // and prioritize certain namespaces over others. + // TODO: Add some more structured types here, this is a crazy map. + let mut grouped_constants: HashMap< + (String, String), + HashMap<u64, Vec<MetadataConstantInfo>>, + > = HashMap::new(); + for constant in &self.constants { + let MetadataTypeKind::Reference { name, namespace } = &constant.ty else { + // TODO: We should optionally provide a way to group constants like these into an enumeration. + // Skipping constant `WDS_MC_TRACE_VERBOSE` with non-reference type `Integer { size: Some(4), is_signed: false }` + // Skipping constant `WDS_MC_TRACE_INFO` with non-reference type `Integer { size: Some(4), is_signed: false }` + // Skipping constant `WDS_MC_TRACE_WARNING` with non-reference type `Integer { size: Some(4), is_signed: false }` + // Skipping constant `WDS_MC_TRACE_ERROR` with non-reference type `Integer { size: Some(4), is_signed: false }` + // Skipping constant `WDS_MC_TRACE_FATAL` with non-reference type `Integer { size: Some(4), is_signed: false }` + tracing::debug!( + "Skipping constant `{}` with non-reference type `{:?}`", + constant.name, + constant.ty + ); + continue; + }; + grouped_constants + .entry((namespace.clone(), name.clone())) + .or_default() + .entry(constant.value) + .or_default() + .push(constant.clone()); + } + + let mut enums = Vec::new(); + for ((enum_namespace, enum_name), mapped_values) in grouped_constants { + let mut variants = Vec::new(); + for (_, group_variants) in mapped_values { + let sorted_group_variants = + sort_metadata_constants_by_proximity(&enum_namespace, group_variants); + let enum_variants: Vec<_> = sorted_group_variants + .iter() + .map(|info| (info.name.clone(), info.value)) + .collect(); + variants.extend(enum_variants); + } + + let enum_kind = MetadataTypeKind::Enum { + ty: Box::new(MetadataTypeKind::Void), + variants, + }; + + enums.push(MetadataTypeInfo { + name: enum_name, + kind: enum_kind, + namespace: enum_namespace, + }); + } + enums + } + + #[allow(dead_code)] + fn update_stale_references(&mut self) { + let mut valid_type_map = HashMap::new(); + for ty in self.types.iter() { + valid_type_map.insert(ty.name.clone(), ty.clone()); + } + + for ty in self.types.iter_mut() { + ty.kind.visit_references_mut(&mut |node| { + let MetadataTypeKind::Reference { name, namespace } = node else { + tracing::error!( + "`visit_references_mut` did not return a reference! {:?}", + node + ); + return; + }; + if let Some(survivor) = valid_type_map.get(name) { + if namespace != &survivor.namespace { + tracing::debug!( + "Updating stale namespace reference `{}` to `{}` for `{}`", + namespace, + survivor.namespace, + name + ); + *namespace = survivor.namespace.clone(); + } + } + }); + } + } +} + +#[derive(Debug, Clone, Eq, Hash, PartialEq)] +pub enum LibraryName { + /// A synthetic library with no associated module name. + /// + /// The shared library is "synthetic" in the sense that a binary view cannot reference it directly. + Namespace(String), + /// A real module with a name (e.g. "info.dll"), these libraries can be referenced directly by a binary view. + Module(String), +} + +#[derive(Debug, Clone, Default)] +pub struct LibraryInfo { + pub metadata: MetadataInfo, + /// A map of externally referenced names to their library names. + /// + /// This is required when resolving type references to other libraries. + pub external_references: HashMap<String, LibraryName>, +} + +#[derive(Debug, Default)] +pub struct PartitionedMetadataInfo { + pub libraries: HashMap<LibraryName, LibraryInfo>, +} + +// TODO: ModuleRef (computable from ModuleInfo and the underlying core module) +// TODO: Put a ModuleRef in all places where a module is associated. +#[derive(Debug, Clone)] +pub struct MetadataModuleInfo { + /// The modules name on disk, this is used to determine the imported + /// function name when loading type information from a type library. + pub name: String, +} + +#[derive(Debug, Clone)] +pub struct MetadataTypeInfo { + pub name: String, + pub kind: MetadataTypeKind, + /// The namespace of the type, e.x. "Windows.Win32.Foundation" + /// + /// This is used to help determine what library this information belongs to. When we go to import + /// this information (along with others), we will build a tree of information where each node + /// corresponds to the namespace, and each child node corresponds to a sub-namespace. Then import + /// info will be enumerated to determine if the type can only ever belong to a single import module + /// if the type is only used in a single module, we will place it in that type library. If the namespace + /// can reference more than one module, we will place it in a common type library named after + /// the namespace itself, it can only ever be referenced by another type library and as such should + /// only contain types and no functions. + /// + /// For more information see [`PartitionedMetadataInfo`]. + pub namespace: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MetadataTypeKind { + Void, + Bool { + // NOTE: Weird optional, if None we actually default the size to integer size! + size: Option<usize>, + }, + Integer { + size: Option<usize>, + is_signed: bool, + }, + Character { + size: usize, + }, + Float { + size: usize, + }, + Pointer { + is_const: bool, + is_pointee_const: bool, + target: Box<MetadataTypeKind>, + }, + Array { + element: Box<MetadataTypeKind>, + count: usize, + }, + Struct { + fields: Vec<MetadataFieldInfo>, + is_packed: bool, + }, + Union { + fields: Vec<MetadataFieldInfo>, + }, + Enum { + ty: Box<MetadataTypeKind>, + variants: Vec<(String, u64)>, + }, + Function { + params: Vec<MetadataParameterInfo>, + return_type: Box<MetadataTypeKind>, + is_vararg: bool, + }, + Reference { + // TODO: Generics may also be passed here. + /// The namespace of the referenced type, e.x. "Windows.Win32.Foundation" + namespace: String, + /// The referenced type name, e.x. "BOOL" + name: String, + }, +} + +impl MetadataTypeKind { + pub(crate) fn visit_references<F>(&self, callback: &mut F) + where + F: FnMut(&str, &str), + { + match self { + MetadataTypeKind::Reference { namespace, name } => { + callback(namespace, name); + } + MetadataTypeKind::Pointer { target, .. } => { + target.visit_references(callback); + } + MetadataTypeKind::Array { element, .. } => { + element.visit_references(callback); + } + MetadataTypeKind::Struct { fields, .. } => { + for field in fields { + field.ty.visit_references(callback); + } + } + MetadataTypeKind::Enum { ty, .. } => { + ty.visit_references(callback); + } + MetadataTypeKind::Function { + params, + return_type, + .. + } => { + for param in params { + param.ty.visit_references(callback); + } + return_type.visit_references(callback); + } + _ => {} + } + } + + #[allow(dead_code)] + pub(crate) fn visit_references_mut<F>(&mut self, callback: &mut F) + where + F: FnMut(&mut MetadataTypeKind), + { + match self { + MetadataTypeKind::Reference { .. } => { + callback(self); + } + MetadataTypeKind::Pointer { target, .. } => { + target.visit_references_mut(callback); + } + MetadataTypeKind::Array { element, .. } => { + element.visit_references_mut(callback); + } + MetadataTypeKind::Struct { fields, .. } | MetadataTypeKind::Union { fields, .. } => { + for field in fields { + field.ty.visit_references_mut(callback); + } + } + MetadataTypeKind::Enum { ty, .. } => { + ty.visit_references_mut(callback); + } + MetadataTypeKind::Function { + params, + return_type, + .. + } => { + for param in params { + param.ty.visit_references_mut(callback); + } + return_type.visit_references_mut(callback); + } + _ => {} + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataFieldInfo { + pub name: String, + pub ty: MetadataTypeKind, + pub is_const: bool, + /// This is only set for bitfields, The first value is the bit position within the associated byte, + /// and the second is the bit width. + /// + /// NOTE: The bit position can never be greater than `7`. + pub bitfield: Option<(u8, u8)>, + // TODO: Attributes ( virtual, static, etc...) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataParameterInfo { + pub name: String, + pub ty: MetadataTypeKind, + // TODO: Attributes (in, out, etc...) +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +pub enum MetadataImportMethod { + ByName(String), + ByOrdinal(u32), +} + +#[derive(Debug, Clone)] +pub struct MetadataImportInfo { + #[allow(dead_code)] + pub method: MetadataImportMethod, + pub module: MetadataModuleInfo, +} + +#[derive(Debug, Clone)] +pub struct MetadataFunctionInfo { + pub name: String, + /// This will only ever be [`MetadataTypeKind::Function`]. + pub ty: MetadataTypeKind, + pub namespace: String, + pub import_info: Option<MetadataImportInfo>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MetadataConstantInfo { + pub name: String, + pub namespace: String, + pub ty: MetadataTypeKind, + pub value: u64, +} + +pub fn sort_metadata_constants_by_proximity( + reference: &str, + mut candidates: Vec<MetadataConstantInfo>, +) -> Vec<MetadataConstantInfo> { + let ref_parts: Vec<&str> = reference.split('.').collect(); + candidates.sort_by_cached_key(|info| { + // Extract the namespace string from the metadata info + let ns = &info.namespace; + let cand_parts = ns.split('.'); + + let score = ref_parts + .iter() + .zip(cand_parts) + .take_while(|(a, b)| *a == b) + .count(); + + // Sort by highest score first, then alphabetically by namespace + (std::cmp::Reverse(score), ns.clone()) + }); + candidates +} diff --git a/plugins/bntl_utils/src/winmd/translate.rs b/plugins/bntl_utils/src/winmd/translate.rs new file mode 100644 index 00000000..01ea54a5 --- /dev/null +++ b/plugins/bntl_utils/src/winmd/translate.rs @@ -0,0 +1,654 @@ +//! Translate windows metadata into a self-contained structure, for later use. + +use super::info::{ + MetadataConstantInfo, MetadataFieldInfo, MetadataFunctionInfo, MetadataImportInfo, + MetadataImportMethod, MetadataInfo, MetadataModuleInfo, MetadataParameterInfo, + MetadataTypeInfo, MetadataTypeKind, +}; +use std::collections::{HashMap, HashSet}; +use thiserror::Error; +use windows_metadata::reader::TypeCategory; +use windows_metadata::{ + AsRow, FieldAttributes, HasAttributes, MethodCallAttributes, Type, TypeAttributes, Value, +}; + +pub const BITFIELD_ATTR: &str = "NativeBitfieldAttribute"; +pub const CONST_ATTR: &str = "ConstAttribute"; +pub const FNPTR_ATTR: &str = "UnmanagedFunctionPointerAttribute"; +pub const _STRUCT_SIZE_ATTR: &str = "StructSizeFieldAttribute"; +pub const API_CONTRACT_ATTR: &str = "ApiContractAttribute"; + +#[derive(Error, Debug)] +pub enum TranslationError { + #[error("no files were provided")] + NoFiles, + #[error("the type name '{0}' is not handled")] + UnhandledType(String), + #[error("the attribute '{0}' is not supported")] + UnsupportedAttribute(String), +} + +pub struct WindowsMetadataTranslator { + // TODO: Allow this to be customized by user. + /// Replace references to a given name with a different one. + /// + /// This allows you to move types to a different namespace or rename them and be certain all + /// references to that type are updated. + remapped_references: HashMap<(&'static str, &'static str), (&'static str, &'static str)>, +} + +impl WindowsMetadataTranslator { + pub fn new() -> Self { + // TODO: Move this to a static array. + let mut remapped_references = HashMap::new(); + remapped_references.insert(("System", "Guid"), ("Windows.Win32.Foundation", "Guid")); + Self { + remapped_references, + } + } + + pub fn translate( + &self, + files: Vec<windows_metadata::reader::File>, + ) -> Result<MetadataInfo, TranslationError> { + if files.is_empty() { + return Err(TranslationError::NoFiles); + } + let index = windows_metadata::reader::TypeIndex::new(files); + self.translate_index(&index) + } + + pub fn translate_index( + &self, + index: &windows_metadata::reader::TypeIndex, + ) -> Result<MetadataInfo, TranslationError> { + let mut functions = Vec::new(); + let mut types = Vec::new(); + let mut constants = Vec::new(); + + // TODO: Move this somewhere else? + // Add synthetic types here. + types.extend([ + MetadataTypeInfo { + name: "Guid".to_string(), + kind: MetadataTypeKind::Struct { + fields: vec![ + MetadataFieldInfo { + name: "Data1".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(4), + is_signed: false, + }, + is_const: false, + bitfield: None, + }, + MetadataFieldInfo { + name: "Data2".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(2), + is_signed: false, + }, + is_const: false, + bitfield: None, + }, + MetadataFieldInfo { + name: "Data3".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(2), + is_signed: false, + }, + is_const: false, + bitfield: None, + }, + MetadataFieldInfo { + name: "Data4".to_string(), + ty: MetadataTypeKind::Array { + element: Box::new(MetadataTypeKind::Integer { + size: Some(1), + is_signed: false, + }), + count: 8, + }, + is_const: false, + bitfield: None, + }, + ], + is_packed: false, + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "HANDLE".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Void), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "HINSTANCE".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Void), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "HMODULE".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Void), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "PCSTR".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: true, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Character { size: 1 }), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "PCWSTR".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: true, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Character { size: 2 }), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "PSTR".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Character { size: 1 }), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "PWSTR".to_string(), + kind: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Character { size: 2 }), + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "UNICODE_STRING".to_string(), + kind: MetadataTypeKind::Struct { + fields: vec![ + MetadataFieldInfo { + name: "Length".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(2), + is_signed: false, + }, + is_const: false, + bitfield: None, + }, + MetadataFieldInfo { + name: "MaximumLength".to_string(), + ty: MetadataTypeKind::Integer { + size: Some(2), + is_signed: false, + }, + is_const: false, + bitfield: None, + }, + MetadataFieldInfo { + name: "Buffer".to_string(), + ty: MetadataTypeKind::Reference { + namespace: "Windows.Win32.Foundation".to_string(), + name: "PWSTR".to_string(), + }, + is_const: false, + bitfield: None, + }, + ], + is_packed: false, + }, + namespace: "Windows.Win32.Foundation".to_string(), + }, + MetadataTypeInfo { + name: "BOOLEAN".to_string(), + kind: MetadataTypeKind::Bool { size: Some(1) }, + namespace: "Windows.Win32.Security".to_string(), + }, + MetadataTypeInfo { + name: "BOOL".to_string(), + // BOOL is integer sized, not char sized like a typical bool value. + kind: MetadataTypeKind::Bool { size: None }, + namespace: "Windows.Win32.Security".to_string(), + }, + ]); + + for entry in index.types() { + match entry.category() { + TypeCategory::Interface => { + let (interface_ty, interface_vtable_ty) = self.translate_interface(&entry)?; + types.push(interface_ty); + types.push(interface_vtable_ty); + } + TypeCategory::Class => { + let (cls_functions, cls_constants) = self.translate_class(&entry)?; + functions.extend(cls_functions); + constants.extend(cls_constants); + } + TypeCategory::Enum => { + types.push(self.translate_enum(&entry)?); + } + TypeCategory::Struct => { + // Skip marker type structures. + if entry.has_attribute(API_CONTRACT_ATTR) { + continue; + } + types.push(self.translate_struct(&entry)?); + } + TypeCategory::Delegate => { + types.push(self.translate_delegate(&entry)?); + } + TypeCategory::Attribute => { + // We will pull attributes directly from the other entries. + } + } + } + + // Remove duplicate types within the same namespace, the first one wins. This is what allows + // us to override types by placing the overrides in the type list before traversing the index. + let mut tracked_names = HashSet::<(String, String)>::new(); + types.retain(|ty| { + let ty_name = (ty.namespace.clone(), ty.name.clone()); + tracked_names.insert(ty_name) + }); + + Ok(MetadataInfo { + types, + functions, + constants, + }) + } + + pub fn translate_struct( + &self, + structure: &windows_metadata::reader::TypeDef, + ) -> Result<MetadataTypeInfo, TranslationError> { + let mut fields = Vec::new(); + + let nested: Result<HashMap<String, _>, _> = structure + .index() + .nested(structure.clone()) + .map(|n| { + // TODO: Are all nested fields a struct? + let nested_ty = self.translate_struct(&n)?; + Ok((n.name().to_string(), nested_ty)) + }) + .collect(); + let nested = nested?; + + for field in structure.fields() { + let mut field_ty = self.translate_type(&field.ty())?; + // TODO: This is kinda ugly. + // Handle nested structures by unwrapping the reference. + let mut nested_ty = None; + field_ty.visit_references(&mut |_, name| { + nested_ty = nested.get(name).cloned().map(|n| n.kind); + }); + field_ty = nested_ty.unwrap_or(field_ty); + + // Bitfields are special, they are a "fake" field that we need to look at the attributes of + // to unwrap the real fields that are contained within the storage type. + if field.has_attribute(BITFIELD_ATTR) { + for bitfield in field.attributes() { + let bitfield_values = bitfield.value(); + let mut values = bitfield_values.iter(); + let Some((_, Value::Utf8(bitfield_name))) = values.next() else { + continue; + }; + let Some((_, Value::I64(bitfield_pos))) = values.next() else { + continue; + }; + let Some((_, Value::I64(bitfield_width))) = values.next() else { + continue; + }; + // is_private, is_public, is_virtual + fields.push(MetadataFieldInfo { + name: bitfield_name.clone(), + ty: field_ty.clone(), + is_const: field.has_attribute(CONST_ATTR), + bitfield: Some((*bitfield_pos as u8, *bitfield_width as u8)), + }); + } + } else { + fields.push(MetadataFieldInfo { + name: field.name().to_string(), + ty: field_ty, + is_const: field.has_attribute(CONST_ATTR), + bitfield: None, + }); + } + } + + let mut is_packed = false; + if let Some(_layout) = structure.class_layout() { + is_packed = _layout.packing_size() == 1; + } + + // ExplicitLayout seems to denote a union layout. + let kind = if structure.flags().contains(TypeAttributes::ExplicitLayout) { + MetadataTypeKind::Union { fields } + } else { + MetadataTypeKind::Struct { fields, is_packed } + }; + + Ok(MetadataTypeInfo { + name: structure.name().to_string(), + kind, + namespace: structure.namespace().to_string(), + }) + } + + pub fn translate_class( + &self, + class: &windows_metadata::reader::TypeDef, + ) -> Result<(Vec<MetadataFunctionInfo>, Vec<MetadataConstantInfo>), TranslationError> { + let namespace = class.namespace().to_string(); + let mut functions = Vec::new(); + for method in class.methods() { + match self.translate_method(&method) { + Ok(mut func) => { + func.namespace = namespace.clone(); + functions.push(func); + } + Err(e) => tracing::warn!("Failed to translate method {}: {}", method.name(), e), + } + } + + let mut constants = Vec::new(); + for field in class.fields() { + if let Some(constant) = field + .constant() + .map(|c| self.value_to_u64(&c.value())) + .flatten() + { + constants.push(MetadataConstantInfo { + name: field.name().to_string(), + namespace: namespace.clone(), + ty: self.translate_type(&field.ty())?, + value: constant, + }); + } else { + tracing::debug!("Field {} is not a constant, skipping...", field.name()); + } + } + + Ok((functions, constants)) + } + + pub fn translate_method( + &self, + method: &windows_metadata::reader::MethodDef, + ) -> Result<MetadataFunctionInfo, TranslationError> { + // TODO: Pass generics here? generic_params seems always empty? Even windows-rs doesn't use it. + let signature = method.signature(&[]); + let func_params: Result<Vec<MetadataParameterInfo>, TranslationError> = method + .params() + .filter(|p| !p.name().is_empty()) + .zip(signature.types) + .map(|(param, param_ty)| { + Ok(MetadataParameterInfo { + name: param.name().to_string(), + ty: self.translate_type(¶m_ty)?, + }) + }) + .collect(); + let func_ty = MetadataTypeKind::Function { + params: func_params?, + return_type: Box::new(self.translate_type(&signature.return_type)?), + is_vararg: signature.flags.contains(MethodCallAttributes::VARARG), + }; + + let import_info = method + .impl_map() + .map(|impl_map| self.import_info_from_map(&impl_map)); + + Ok(MetadataFunctionInfo { + name: method.name().to_string(), + ty: func_ty, + // NOTE: This will be set by the associated class entry once returned. + namespace: "".to_string(), + import_info, + }) + } + + pub fn translate_delegate( + &self, + delegate: &windows_metadata::reader::TypeDef, + ) -> Result<MetadataTypeInfo, TranslationError> { + if !delegate.has_attribute(FNPTR_ATTR) { + return Err(TranslationError::UnsupportedAttribute( + FNPTR_ATTR.to_string(), + )); + } + let invoke_method = delegate + .methods() + .find(|m| m.name() == "Invoke") + .expect("Invoke method not found"); + let translated_invoke_method = self.translate_method(&invoke_method)?; + Ok(MetadataTypeInfo { + name: delegate.name().to_string(), + kind: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(translated_invoke_method.ty), + }, + namespace: delegate.namespace().to_string(), + }) + } + + pub fn translate_interface( + &self, + interface: &windows_metadata::reader::TypeDef, + ) -> Result<(MetadataTypeInfo, MetadataTypeInfo), TranslationError> { + let mut vtable_fields = Vec::new(); + for meth in interface.methods() { + let meth_ty = self.translate_method(&meth)?; + vtable_fields.push(MetadataFieldInfo { + name: meth.name().to_string(), + ty: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(meth_ty.ty), + }, + is_const: false, + bitfield: None, + }) + } + + let interface_ns = interface.namespace(); + let interface_ty = MetadataTypeInfo { + name: interface.name().to_string(), + kind: MetadataTypeKind::Struct { + fields: vec![MetadataFieldInfo { + name: "vtable".to_string(), + ty: MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(MetadataTypeKind::Reference { + namespace: interface_ns.to_string(), + name: format!("{}VTable", interface.name()), + }), + }, + is_const: false, + bitfield: None, + }], + is_packed: false, + }, + namespace: interface_ns.to_string(), + }; + let interface_vtable_ty = MetadataTypeInfo { + name: format!("{}VTable", interface.name()), + kind: MetadataTypeKind::Struct { + fields: Vec::new(), + is_packed: false, + }, + namespace: interface_ns.to_string(), + }; + Ok((interface_ty, interface_vtable_ty)) + } + + pub fn translate_enum( + &self, + _enum: &windows_metadata::reader::TypeDef, + ) -> Result<MetadataTypeInfo, TranslationError> { + let mut variants = Vec::new(); + let mut last_constant = 0; + let mut enum_ty = MetadataTypeKind::Integer { + size: None, + is_signed: true, + }; + for variant in _enum.fields() { + if variant.flags().contains(FieldAttributes::RTSpecialName) { + // Skip the hidden "value__" field. + continue; + } + // Pull the enums type from the constant if it exists. + // Otherwise, we will fall back to void and use a default type when importing. + if let Some(constant) = variant.constant() { + enum_ty = self.translate_type(&constant.ty())?; + } + let variant_constant = variant + .constant() + .map(|c| self.value_to_u64(&c.value())) + .flatten() + .unwrap_or(last_constant); + let variant_name = variant.name().to_string(); + variants.push((variant_name, variant_constant)); + last_constant = variant_constant; + } + Ok(MetadataTypeInfo { + name: _enum.name().to_string(), + kind: MetadataTypeKind::Enum { + ty: Box::new(enum_ty), + variants, + }, + namespace: _enum.namespace().to_string(), + }) + } + + pub fn translate_type(&self, ty: &Type) -> Result<MetadataTypeKind, TranslationError> { + match ty { + Type::Void => Ok(MetadataTypeKind::Void), + Type::Bool => Ok(MetadataTypeKind::Bool { size: Some(1) }), + Type::Char => Ok(MetadataTypeKind::Character { size: 1 }), + Type::I8 => Ok(MetadataTypeKind::Integer { + size: Some(1), + is_signed: true, + }), + Type::U8 => Ok(MetadataTypeKind::Integer { + size: Some(1), + is_signed: false, + }), + Type::I16 => Ok(MetadataTypeKind::Integer { + size: Some(2), + is_signed: true, + }), + Type::U16 => Ok(MetadataTypeKind::Integer { + size: Some(2), + is_signed: false, + }), + Type::I32 => Ok(MetadataTypeKind::Integer { + size: Some(4), + is_signed: true, + }), + Type::U32 => Ok(MetadataTypeKind::Integer { + size: Some(4), + is_signed: false, + }), + Type::I64 => Ok(MetadataTypeKind::Integer { + size: Some(8), + is_signed: true, + }), + Type::U64 => Ok(MetadataTypeKind::Integer { + size: Some(8), + is_signed: false, + }), + Type::F32 => Ok(MetadataTypeKind::Float { size: 4 }), + Type::F64 => Ok(MetadataTypeKind::Float { size: 8 }), + Type::ISize => Ok(MetadataTypeKind::Integer { + size: None, + is_signed: true, + }), + Type::USize => Ok(MetadataTypeKind::Integer { + size: None, + is_signed: false, + }), + Type::Name(name) => { + if let Some((remapped_ns, remapped_name)) = + self.remapped_references.get(&(&name.namespace, &name.name)) + { + Ok(MetadataTypeKind::Reference { + namespace: remapped_ns.to_string(), + name: remapped_name.to_string(), + }) + } else { + Ok(MetadataTypeKind::Reference { + namespace: name.namespace.clone(), + name: name.name.clone(), + }) + } + } + Type::PtrMut(target, _) => Ok(MetadataTypeKind::Pointer { + is_const: false, + is_pointee_const: false, + target: Box::new(self.translate_type(target)?), + }), + Type::PtrConst(target, _) => { + Ok(MetadataTypeKind::Pointer { + is_const: false, + // TODO: I think this might be pointee const? + is_pointee_const: true, + target: Box::new(self.translate_type(target)?), + }) + } + Type::ArrayFixed(elem_ty, count) => Ok(MetadataTypeKind::Array { + element: Box::new(self.translate_type(elem_ty)?), + count: *count, + }), + other => Err(TranslationError::UnhandledType(format!("{:?}", other))), + } + } + + pub fn import_info_from_map( + &self, + map: &windows_metadata::reader::ImplMap, + ) -> MetadataImportInfo { + MetadataImportInfo { + method: MetadataImportMethod::ByName(map.import_name().to_string()), + module: MetadataModuleInfo { + name: map.import_scope().name().to_string(), + }, + } + } + + pub fn value_to_u64(&self, value: &Value) -> Option<u64> { + match value { + Value::Bool(b) => Some(*b as u64), + Value::U8(i) => Some(*i as u64), + Value::I8(i) => Some(*i as u64), + Value::U16(i) => Some(*i as u64), + Value::I16(i) => Some(*i as u64), + Value::U32(i) => Some(*i as u64), + Value::I32(i) => Some(*i as u64), + Value::U64(i) => Some(*i), + Value::I64(i) => Some(*i as u64), + _ => None, + } + } +} |
