summaryrefslogtreecommitdiff
path: root/plugins
diff options
context:
space:
mode:
authorMason Reed <mason@vector35.com>2025-07-13 14:39:14 -0400
committerMason Reed <mason@vector35.com>2025-07-15 12:34:43 -0400
commit4a49ba509bdc0b4ffa650fc8461738c2085161c7 (patch)
tree0c55d84afde06cfe1416f8f199df985b41a0f1ff /plugins
parent9c80b724bc28eca0b1b4fad1b955d4fd2ba48ab5 (diff)
[WARP] Add network container
This is going to be disabled by default on this upcoming stable, however users may enable it once we deploy the public server. The data from the server is done through `Container::fetch_functions` independent of the nonblocking function lookup functions. The sidebar has been updated to drive fetching so that when users navigate to a new function the fetcher will kick off. This fetcher operates on a separate thread, in the event of a user navigating to many functions before the current fetch has completed they all will be batched together in a single fetch. Networked container currently is limited to just function prototypes, other type information separate from the function object will be omitted.
Diffstat (limited to 'plugins')
-rw-r--r--plugins/warp/Cargo.toml1
-rw-r--r--plugins/warp/api/python/_warpcore.py23
-rw-r--r--plugins/warp/api/python/warp.py7
-rw-r--r--plugins/warp/api/warp.cpp10
-rw-r--r--plugins/warp/api/warp.h2
-rw-r--r--plugins/warp/api/warpcore.h2
-rw-r--r--plugins/warp/src/container.rs13
-rw-r--r--plugins/warp/src/container/network.rs332
-rw-r--r--plugins/warp/src/container/network/client.rs228
-rw-r--r--plugins/warp/src/plugin.rs89
-rw-r--r--plugins/warp/src/plugin/ffi/container.rs21
-rw-r--r--plugins/warp/src/plugin/settings.rs28
-rw-r--r--plugins/warp/src/plugin/workflow.rs17
-rw-r--r--plugins/warp/ui/matches.cpp71
-rw-r--r--plugins/warp/ui/matches.h9
-rw-r--r--plugins/warp/ui/plugin.cpp3
16 files changed, 812 insertions, 44 deletions
diff --git a/plugins/warp/Cargo.toml b/plugins/warp/Cargo.toml
index afebf577..761ac961 100644
--- a/plugins/warp/Cargo.toml
+++ b/plugins/warp/Cargo.toml
@@ -24,6 +24,7 @@ thiserror = "2.0"
ar = { git = "https://github.com/mdsteele/rust-ar" }
tempdir = "0.3.7"
regex = "1.11"
+reqwest = { version = "0.12", features = ["blocking", "json", "multipart"] }
# For reports
minijinja = "2.10.2"
diff --git a/plugins/warp/api/python/_warpcore.py b/plugins/warp/api/python/_warpcore.py
index 740072ab..da262f18 100644
--- a/plugins/warp/api/python/_warpcore.py
+++ b/plugins/warp/api/python/_warpcore.py
@@ -205,6 +205,29 @@ def BNWARPContainerCommitSource(
# -------------------------------------------------------
+# _BNWARPContainerFetchFunctions
+
+_BNWARPContainerFetchFunctions = core.BNWARPContainerFetchFunctions
+_BNWARPContainerFetchFunctions.restype = None
+_BNWARPContainerFetchFunctions.argtypes = [
+ ctypes.POINTER(BNWARPContainer),
+ ctypes.POINTER(BNWARPTarget),
+ ctypes.POINTER(BNWARPTypeGUID),
+ ctypes.c_ulonglong,
+ ]
+
+
+# noinspection PyPep8Naming
+def BNWARPContainerFetchFunctions(
+ container: ctypes.POINTER(BNWARPContainer),
+ target: ctypes.POINTER(BNWARPTarget),
+ guids: ctypes.POINTER(BNWARPTypeGUID),
+ count: int
+ ) -> None:
+ return _BNWARPContainerFetchFunctions(container, target, guids, count)
+
+
+# -------------------------------------------------------
# _BNWARPContainerGetFunctionsWithGUID
_BNWARPContainerGetFunctionsWithGUID = core.BNWARPContainerGetFunctionsWithGUID
diff --git a/plugins/warp/api/python/warp.py b/plugins/warp/api/python/warp.py
index c761c323..c7486b8c 100644
--- a/plugins/warp/api/python/warp.py
+++ b/plugins/warp/api/python/warp.py
@@ -305,6 +305,13 @@ class WarpContainer(metaclass=_WarpContainerMetaclass):
core_guids[i] = guids[i].uuid
return warpcore.BNWARPContainerRemoveTypes(self.handle, source.uuid, core_guids, count)
+ def fetch_functions(self, target: WarpTarget, guids: List[FunctionGUID]):
+ count = len(guids)
+ core_guids = (warpcore.BNWARPFunctionGUID * count)()
+ for i in range(count):
+ core_guids[i] = guids[i].uuid
+ warpcore.BNWARPContainerFetchFunctions(self.handle, target.handle, core_guids, count)
+
def get_sources_with_function_guid(self, target: WarpTarget, guid: FunctionGUID) -> List[Source]:
count = ctypes.c_size_t()
sources = warpcore.BNWARPContainerGetSourcesWithFunctionGUID(self.handle, target.handle, guid.uuid, count)
diff --git a/plugins/warp/api/warp.cpp b/plugins/warp/api/warp.cpp
index 147add8e..52c7ea9d 100644
--- a/plugins/warp/api/warp.cpp
+++ b/plugins/warp/api/warp.cpp
@@ -249,6 +249,16 @@ bool Container::RemoveTypes(const Source &source, const std::vector<TypeGUID> &g
return result;
}
+void Container::FetchFunctions(const Target &target, const std::vector<FunctionGUID> &guids) const
+{
+ size_t count = guids.size();
+ BNWARPFunctionGUID *apiGuids = new BNWARPFunctionGUID[count];
+ for (size_t i = 0; i < count; i++)
+ apiGuids[i] = *guids[i].Raw();
+ BNWARPContainerFetchFunctions(m_object, target.m_object, apiGuids, count);
+ delete[] apiGuids;
+}
+
std::vector<Source> Container::GetSourcesWithFunctionGUID(const Target& target, const FunctionGUID &guid) const
{
size_t count;
diff --git a/plugins/warp/api/warp.h b/plugins/warp/api/warp.h
index b57a0c7b..807ba998 100644
--- a/plugins/warp/api/warp.h
+++ b/plugins/warp/api/warp.h
@@ -354,6 +354,8 @@ namespace Warp {
bool RemoveTypes(const Source &source, const std::vector<TypeGUID> &guids) const;
+ void FetchFunctions(const Target& target, const std::vector<FunctionGUID> &guids) const;
+
std::vector<Source> GetSourcesWithFunctionGUID(const Target& target, const FunctionGUID &guid) const;
std::vector<Source> GetSourcesWithTypeGUID(const TypeGUID &guid) const;
diff --git a/plugins/warp/api/warpcore.h b/plugins/warp/api/warpcore.h
index 09190126..20a59b7b 100644
--- a/plugins/warp/api/warpcore.h
+++ b/plugins/warp/api/warpcore.h
@@ -108,6 +108,8 @@ extern "C"
WARP_FFI_API bool BNWARPContainerRemoveFunctions(BNWARPContainer* container, const BNWARPTarget* target, const BNWARPSource* source, BNWARPFunction** functions, size_t count);
WARP_FFI_API bool BNWARPContainerRemoveTypes(BNWARPContainer* container, const BNWARPSource* source, BNWARPTypeGUID* types, size_t count);
+ WARP_FFI_API void BNWARPContainerFetchFunctions(BNWARPContainer* container, BNWARPTarget* target, const BNWARPTypeGUID* guids, size_t count);
+
WARP_FFI_API BNWARPSource* BNWARPContainerGetSourcesWithFunctionGUID(BNWARPContainer* container, const BNWARPTarget* target, const BNWARPFunctionGUID* guid, size_t* count);
WARP_FFI_API BNWARPSource* BNWARPContainerGetSourcesWithTypeGUID(BNWARPContainer* container, const BNWARPTypeGUID* guid, size_t* count);
WARP_FFI_API BNWARPFunction** BNWARPContainerGetFunctionsWithGUID(BNWARPContainer* container, const BNWARPTarget* target, const BNWARPSource* source, const BNWARPFunctionGUID* guid, size_t* count);
diff --git a/plugins/warp/src/container.rs b/plugins/warp/src/container.rs
index 8d720021..4cabc60e 100644
--- a/plugins/warp/src/container.rs
+++ b/plugins/warp/src/container.rs
@@ -208,6 +208,19 @@ pub trait Container: Send + Sync + Display + Debug {
functions: &[Function],
) -> ContainerResult<()>;
+ /// Fetches WARP information for the associated functions.
+ ///
+ /// Typically, a container that resides only in memory has nothing to fetch, so the default implementation
+ /// will do nothing. This function is blocking, so assume it will take a few seconds for a container
+ /// that intends to fetch over the network.
+ fn fetch_functions(
+ &mut self,
+ _target: &Target,
+ _functions: &[FunctionGUID],
+ ) -> ContainerResult<()> {
+ Ok(())
+ }
+
/// Get the sources that contain a type with the given [`TypeGUID`].
fn sources_with_type_guid(&self, guid: &TypeGUID) -> ContainerResult<Vec<SourceId>>;
diff --git a/plugins/warp/src/container/network.rs b/plugins/warp/src/container/network.rs
index ffbe6108..2a2d7c65 100644
--- a/plugins/warp/src/container/network.rs
+++ b/plugins/warp/src/container/network.rs
@@ -1,13 +1,323 @@
-pub struct NetworkContainer {}
+use crate::container::disk::DiskContainer;
+use crate::container::{Container, ContainerError, ContainerResult, SourceId, SourcePath};
+use std::collections::HashMap;
+use std::fmt::{Debug, Display, Formatter};
+use warp::chunk::{Chunk, ChunkKind, CompressionType};
+use warp::r#type::guid::TypeGUID;
+use warp::r#type::{ComputedType, Type};
+use warp::signature::chunk::SignatureChunk;
+use warp::signature::function::{Function, FunctionGUID};
+use warp::target::Target;
+use warp::{WarpFile, WarpFileHeader};
-// TODO: The container is populated as the user is navigating a binary
-// TODO: We need to have a few helper functions here to post and pull
-// TODO: Then in the interface we operate off the network cache
-// TODO: The network cache could just be a disk container? Or disk container sources?
-// TODO: We should also store the cache on the filesystem for a certain time, will need to timestamp
-// TODO: When we commit we need to actually POST i believe.
-// TODO: There needs to be a setting that adjusts the sweep size of functions at the cursor.
-// TODO: Probably need a callback or something to tell the network containers to refresh from the network.
-// TODO: The network container should never instantiate itself, unless its gurenteed to not have any data in it?
+pub mod client;
-// TODO: Need to PUSH chunks and PULL chunks
+pub use client::NetworkClient;
+
+/// This is the id on the server for the [`Target`], we can get it via [`NetworkClient::query_target_id`].
+pub type NetworkTargetId = i32;
+
+pub struct NetworkContainer {
+ client: NetworkClient,
+ /// This is the store that the interface will write to; then we have special functions for pulling
+ /// and pushing to the network source.
+ cache: DiskContainer,
+ /// Populated when targets are queried.
+ known_targets: HashMap<Target, Option<NetworkTargetId>>,
+ /// Populated with function sources are queried.
+ known_function_sources: HashMap<FunctionGUID, Vec<SourceId>>,
+ /// Populated when user adds function, this is used for writing back to the server.
+ added_chunks: HashMap<SourceId, Vec<Chunk<'static>>>,
+}
+
+impl NetworkContainer {
+ pub fn new(client: NetworkClient) -> Self {
+ Self {
+ cache: DiskContainer::new("Network Container".to_string(), HashMap::new()),
+ client,
+ known_targets: HashMap::new(),
+ known_function_sources: HashMap::new(),
+ added_chunks: HashMap::new(),
+ }
+ }
+
+ /// Gets the network id for the `target`, this will be used in later function queries.
+ ///
+ /// **This is blocking**
+ ///
+ /// # Caching policy
+ ///
+ /// The [`NetworkTargetId`] is unique and immutable, so they will be persisted indefinitely.
+ pub fn get_target_id(&mut self, target: &Target) -> Option<NetworkTargetId> {
+ // It's highly probable we have previously queried the target, check that first.
+ if let Some(target_id) = self.known_targets.get(target) {
+ return target_id.clone();
+ }
+
+ let target_id = self.client.query_target_id(target);
+ // Keep the target id so the next lookup is free.
+ self.known_targets.insert(target.clone(), target_id);
+ target_id
+ }
+
+ /// Pulls sources for the set of unseen function guids.
+ ///
+ /// **This is blocking**
+ ///
+ /// # Caching policy
+ ///
+ /// When we get the source, we store the results indefinitely in the container; this is fine
+ /// for now as the requests for functions come at the request of some user interaction. Any guid
+ /// with no sources will still be cached.
+ pub fn get_unseen_functions_source(
+ &mut self,
+ target: Option<&Target>,
+ guids: &[FunctionGUID],
+ ) -> HashMap<SourceId, Vec<FunctionGUID>> {
+ let Some(target_id) = target.and_then(|t| self.get_target_id(t)) else {
+ log::debug!("Cannot query functions source without a target, skipping...");
+ return HashMap::new();
+ };
+
+ // Split guids into known and unknown
+ let (_known, unknown): (Vec<_>, Vec<_>) = guids
+ .into_iter()
+ .cloned()
+ .partition(|guid| self.known_function_sources.contains_key(guid));
+
+ let mut result: HashMap<SourceId, Vec<FunctionGUID>> = HashMap::new();
+ // Only query server for unknown guids if we have any.
+ if !unknown.is_empty() {
+ if let Some(queried_results) = self
+ .client
+ .query_functions_source(Some(target_id), &unknown)
+ {
+ // Cache the new results, this means we will not try and contact the server for that guids source.
+ // NOTE: Here we do not just simply list the queried results because we also
+ // want to cache function guids which have no source, this is important so that we never
+ // attempt to contact the server for that guid.
+ for guid in &unknown {
+ let sources = queried_results
+ .keys()
+ .filter(|source_id| queried_results[source_id].contains(guid))
+ .cloned()
+ .collect();
+ self.known_function_sources.insert(*guid, sources);
+ }
+
+ for (source_id, guids) in queried_results {
+ result.entry(source_id).or_default().extend(guids);
+ }
+ }
+ }
+
+ result
+ }
+
+ /// Pulls function metadata from the server and adds it into the container cache.
+ ///
+ /// **This is blocking**
+ ///
+ /// # Caching policy
+ ///
+ /// Every request we store the returned objects on disk, this means that users will first
+ /// query against the disk objects, then the server. This also means we need to cache functions f
+ /// or which we have not received any functions for, as otherwise we would keep trying to query it.
+ pub fn pull_functions(
+ &mut self,
+ target: &Target,
+ source: &SourceId,
+ functions: &[FunctionGUID],
+ ) {
+ let target_id = self.get_target_id(target);
+ if let Some(file) = self
+ .client
+ .query_functions(target_id, Some(*source), functions)
+ {
+ log::debug!("Got {} chunks from server", file.chunks.len());
+ for chunk in &file.chunks {
+ match &chunk.kind {
+ ChunkKind::Signature(sc) => {
+ let functions: Vec<_> = sc.functions().collect();
+ match self.cache.add_functions(target, source, &functions) {
+ Ok(_) => log::debug!(
+ "Added {} functions into cached source '{}'",
+ functions.len(),
+ source
+ ),
+ Err(err) => log::error!(
+ "Failed to add {} function into cached source '{}': {}",
+ functions.len(),
+ source,
+ err
+ ),
+ }
+ }
+ // TODO; Probably want to pull type in with this.
+ ChunkKind::Type(_) => {}
+ }
+ }
+ }
+ }
+
+ /// Push a file to the network source.
+ ///
+ /// **This is blocking**
+ pub fn push_file(&mut self, source_id: SourceId, file: &WarpFile) {
+ self.client.push_file(source_id, file);
+ }
+}
+
+impl Container for NetworkContainer {
+ fn sources(&self) -> ContainerResult<Vec<SourceId>> {
+ self.cache.sources()
+ }
+
+ fn add_source(&mut self, path: SourcePath) -> ContainerResult<SourceId> {
+ // TODO: How do we want to let users create new sources?
+ log::error!("NetworkContainer::add_source not allowed");
+ Err(ContainerError::CannotCreateSource(path))
+ }
+
+ fn commit_source(&mut self, source: &SourceId) -> ContainerResult<bool> {
+ let chunks = self
+ .added_chunks
+ .remove(source)
+ .ok_or(ContainerError::SourceNotFound(source.clone()))?;
+ let file = WarpFile::new(WarpFileHeader::new(), chunks);
+ self.push_file(*source, &file);
+ Ok(true)
+ }
+
+ fn is_source_writable(&self, source: &SourceId) -> ContainerResult<bool> {
+ // TODO: This is retrievable from /users/me/sources we will grab it when connecting.
+ log::error!("NetworkContainer::is_source_writable not allowed");
+ Err(ContainerError::SourceNotWritable(source.clone()))
+ }
+
+ fn is_source_uncommitted(&self, source: &SourceId) -> ContainerResult<bool> {
+ Ok(self.added_chunks.contains_key(source))
+ }
+
+ fn source_path(&self, source: &SourceId) -> ContainerResult<SourcePath> {
+ self.cache.source_path(source)
+ }
+
+ fn add_computed_types(
+ &mut self,
+ source: &SourceId,
+ types: &[ComputedType],
+ ) -> ContainerResult<()> {
+ self.cache.add_computed_types(source, types)
+ }
+
+ fn remove_types(&mut self, source: &SourceId, guids: &[TypeGUID]) -> ContainerResult<()> {
+ self.cache.remove_types(source, guids)
+ }
+
+ fn add_functions(
+ &mut self,
+ target: &Target,
+ source: &SourceId,
+ functions: &[Function],
+ ) -> ContainerResult<()> {
+ let signature_chunk = SignatureChunk::new(functions).ok_or(
+ ContainerError::CorruptedData("signature chunk failed to validate"),
+ )?;
+ let chunk = Chunk::new_with_target(
+ ChunkKind::Signature(signature_chunk),
+ CompressionType::None,
+ target.clone(),
+ );
+ self.added_chunks.entry(*source).or_default().push(chunk);
+ Ok(())
+ }
+
+ fn remove_functions(
+ &mut self,
+ target: &Target,
+ source: &SourceId,
+ functions: &[Function],
+ ) -> ContainerResult<()> {
+ // TODO: Wont persist, need to add remote removal.
+ self.cache.remove_functions(target, source, functions)
+ }
+
+ fn fetch_functions(
+ &mut self,
+ target: &Target,
+ functions: &[FunctionGUID],
+ ) -> ContainerResult<()> {
+ // NOTE: Blocking request to get the mapped function sources.
+ let mapped_unseen_functions = self.get_unseen_functions_source(Some(&target), functions);
+
+ // Actually get the function data for the unseen guids, we really only want to do this once per
+ // session, anymore, and this is annoying!
+ for (source, unseen_guids) in mapped_unseen_functions {
+ // NOTE: Blocking request to get the function data in the container cache.
+ self.pull_functions(&target, &source, &unseen_guids);
+ }
+
+ Ok(())
+ }
+
+ fn sources_with_type_guid(&self, guid: &TypeGUID) -> ContainerResult<Vec<SourceId>> {
+ self.cache.sources_with_type_guid(guid)
+ }
+
+ fn sources_with_type_guids(
+ &self,
+ guids: &[TypeGUID],
+ ) -> ContainerResult<HashMap<TypeGUID, Vec<SourceId>>> {
+ self.cache.sources_with_type_guids(guids)
+ }
+
+ fn type_guids_with_name(
+ &self,
+ source: &SourceId,
+ name: &str,
+ ) -> ContainerResult<Vec<TypeGUID>> {
+ self.cache.type_guids_with_name(source, name)
+ }
+
+ fn type_with_guid(&self, source: &SourceId, guid: &TypeGUID) -> ContainerResult<Option<Type>> {
+ self.cache.type_with_guid(source, guid)
+ }
+
+ fn sources_with_function_guid(
+ &self,
+ target: &Target,
+ guid: &FunctionGUID,
+ ) -> ContainerResult<Vec<SourceId>> {
+ self.cache.sources_with_function_guid(target, guid)
+ }
+
+ fn sources_with_function_guids(
+ &self,
+ target: &Target,
+ guids: &[FunctionGUID],
+ ) -> ContainerResult<HashMap<FunctionGUID, Vec<SourceId>>> {
+ self.cache.sources_with_function_guids(target, guids)
+ }
+
+ fn functions_with_guid(
+ &self,
+ target: &Target,
+ source: &SourceId,
+ guid: &FunctionGUID,
+ ) -> ContainerResult<Vec<Function>> {
+ self.cache.functions_with_guid(target, source, guid)
+ }
+}
+
+impl Debug for NetworkContainer {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("NetworkContainer").finish()
+ }
+}
+
+impl Display for NetworkContainer {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("NetworkContainer").finish()
+ }
+}
diff --git a/plugins/warp/src/container/network/client.rs b/plugins/warp/src/container/network/client.rs
new file mode 100644
index 00000000..f77f1118
--- /dev/null
+++ b/plugins/warp/src/container/network/client.rs
@@ -0,0 +1,228 @@
+use crate::container::network::NetworkTargetId;
+use crate::container::SourceId;
+use reqwest::blocking::Client;
+use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION};
+use reqwest::StatusCode;
+use serde_json::json;
+use std::collections::HashMap;
+use std::str::FromStr;
+use warp::signature::function::FunctionGUID;
+use warp::target::Target;
+use warp::WarpFile;
+
+/// Responsible for sending and receiving data from the server.
+///
+/// NOTE: **All requests are blocking**.
+#[derive(Clone, Debug)]
+pub struct NetworkClient {
+ client: Client,
+ server_url: String,
+}
+
+impl NetworkClient {
+ pub fn new(
+ server_url: String,
+ server_token: Option<String>,
+ https_proxy: Option<String>,
+ ) -> reqwest::Result<Self> {
+ let version_info = binaryninja::version_info();
+ // TODO: IIRC we had a user agent format already for some other thing.
+ let client_agent = format!(
+ "Binary Ninja/{}.{}.{}",
+ version_info.major, version_info.minor, version_info.build
+ );
+ // TODO: This might want to be kept for the request header?
+ let mut headers = HeaderMap::new();
+ if let Some(token) = &server_token {
+ headers.insert(
+ AUTHORIZATION,
+ HeaderValue::from_str(&format!("Bearer {}", token)).unwrap(),
+ );
+ }
+ // TODO: Configurable timeout?
+ let mut client_builder = Client::builder()
+ .connect_timeout(std::time::Duration::from_secs(10))
+ .default_headers(headers)
+ .user_agent(client_agent);
+ if let Some(https_proxy) = https_proxy {
+ client_builder = client_builder.proxy(reqwest::Proxy::all(&https_proxy)?);
+ }
+ Ok(Self {
+ client: client_builder.build()?,
+ server_url,
+ })
+ }
+
+ /// Check to see the status of the server.
+ ///
+ /// This is useful if you want to fail early and prevent constructing a network container to a
+ /// server that is unresponsive.
+ ///
+ /// Route: `api/v1/status`
+ pub fn status(&self) -> reqwest::Result<StatusCode> {
+ let status_url = format!("{}/api/v1/status", self.server_url);
+ let resp = self.client.get(&status_url).send()?;
+ Ok(resp.status())
+ }
+
+ /// Query the [`NetworkTargetId`] for the given [`Target`].
+ ///
+ /// NOTE: **THIS IS BLOCKING**
+ ///
+ /// Route: `api/v1/targets/query` (TODO: Comment about the query)
+ pub fn query_target_id(&self, target: &Target) -> Option<NetworkTargetId> {
+ let query_target_url = format!("{}/api/v1/targets/query", self.server_url);
+
+ let mut query = HashMap::new();
+ if let Some(platform) = &target.platform {
+ query.insert("platform", platform);
+ }
+ if let Some(architecture) = &target.architecture {
+ query.insert("architecture", architecture);
+ }
+
+ // NOTE: This is blocking.
+ let target_id: NetworkTargetId = self
+ .client
+ .get(query_target_url)
+ .query(&query)
+ .send()
+ .ok()?
+ .json::<NetworkTargetId>()
+ .ok()?;
+
+ Some(target_id)
+ }
+
+ fn query_functions_body(
+ target: Option<NetworkTargetId>,
+ source: Option<SourceId>,
+ guids: &[FunctionGUID],
+ ) -> serde_json::Value {
+ let guids_str: Vec<String> = guids.iter().map(|g| g.to_string()).collect();
+ // TODO: The limit here needs to be somewhat flexible. But 1000 will do for now.
+ let mut body = json!({
+ "format": "flatbuffer",
+ "guids": guids_str,
+ "limit": 1000
+ });
+ if let Some(target_id) = target {
+ body["target_id"] = json!(target_id);
+ }
+ if let Some(source_id) = source {
+ body["source_id"] = json!(source_id.to_string());
+ }
+ body
+ }
+
+ /// Query the functions, returning the warp file response containing the entries.
+ ///
+ /// NOTE: **THIS IS BLOCKING**
+ ///
+ /// Route: `api/v1/functions/query` (TODO: Comment about the query)
+ pub fn query_functions(
+ &self,
+ target: Option<NetworkTargetId>,
+ source: Option<SourceId>,
+ guids: &[FunctionGUID],
+ ) -> Option<WarpFile<'static>> {
+ let query_functions_url = format!("{}/api/v1/functions/query", self.server_url);
+ let payload = Self::query_functions_body(target, source, guids);
+
+ // Make the POST request
+ let response = self
+ .client
+ .post(&query_functions_url)
+ .json(&payload)
+ .send()
+ .ok()?;
+ if !response.status().is_success() {
+ log::error!("Failed to query functions: {}", response.status());
+ return None;
+ }
+
+ // Get response bytes and convert to WarpFile
+ let bytes = response.bytes().ok()?;
+ WarpFile::from_owned_bytes(bytes.to_vec())
+ }
+
+ /// Query the functions, returning the sources and the corresponding function guids.
+ ///
+ /// NOTE: **THIS IS BLOCKING**
+ ///
+ /// Route: `api/v1/functions/query/source` (TODO: Comment about the query)
+ pub fn query_functions_source(
+ &self,
+ target: Option<NetworkTargetId>,
+ guids: &[FunctionGUID],
+ ) -> Option<HashMap<SourceId, Vec<FunctionGUID>>> {
+ let query_functions_source_url =
+ format!("{}/api/v1/functions/query/source", self.server_url);
+ let payload = Self::query_functions_body(target, None, guids);
+
+ // Make the POST request
+ let response = self
+ .client
+ .post(&query_functions_source_url)
+ .json(&payload)
+ .send()
+ .ok()?;
+ if !response.status().is_success() {
+ log::error!("Failed to query functions source: {}", response.status());
+ return None;
+ }
+
+ // Mapping of source id to function guids
+ let json_response: HashMap<String, Vec<String>> = response.json().ok()?;
+ let mapped_function_guids = json_response
+ .into_iter()
+ .filter_map(|(source_str, guid_strs)| {
+ let source_id = SourceId::from_str(&source_str).ok()?;
+ let guids = guid_strs
+ .into_iter()
+ .filter_map(|guid_str| FunctionGUID::from_str(&guid_str).ok())
+ .collect();
+ Some((source_id, guids))
+ })
+ .collect();
+
+ Some(mapped_function_guids)
+ }
+
+ /// Pushes the file to the remote source.
+ ///
+ /// NOTE: **THIS IS BLOCKING**
+ ///
+ /// Route: `api/v1/files/{source}`
+ pub fn push_file(&self, source_id: SourceId, file: &WarpFile) -> bool {
+ let push_file_url = format!("{}/api/v1/files/{}", self.server_url, source_id.to_string());
+
+ // Convert WarpFile to bytes
+ let file_bytes = file.to_bytes();
+
+ // Create the form part with the file
+ let form = reqwest::blocking::multipart::Form::new().part(
+ "file",
+ reqwest::blocking::multipart::Part::bytes(file_bytes)
+ .file_name("data.warp")
+ .mime_str("application/octet-stream")
+ .unwrap(),
+ );
+
+ // Send the request
+ match self.client.post(&push_file_url).multipart(form).send() {
+ Ok(response) => {
+ if response.status().is_success() {
+ true
+ } else {
+ log::error!("Failed to push file: {}", response.status());
+ false
+ }
+ }
+ Err(e) => {
+ log::error!("Failed to send push request: {}", e);
+ false
+ }
+ }
+ }
+}
diff --git a/plugins/warp/src/plugin.rs b/plugins/warp/src/plugin.rs
index 3380c619..7f0481b8 100644
--- a/plugins/warp/src/plugin.rs
+++ b/plugins/warp/src/plugin.rs
@@ -3,6 +3,7 @@ use std::time::Instant;
use crate::cache::container::add_cached_container;
use crate::container::disk::DiskContainer;
+use crate::container::network::{NetworkClient, NetworkContainer};
use crate::matcher::MatcherSettings;
use crate::plugin::render_layer::HighlightRenderLayer;
use crate::plugin::settings::PluginSettings;
@@ -11,9 +12,11 @@ use binaryninja::background_task::BackgroundTask;
use binaryninja::command::{
register_command, register_command_for_function, register_command_for_project,
};
+use binaryninja::is_ui_enabled;
use binaryninja::logger::Logger;
use binaryninja::settings::Settings;
use log::LevelFilter;
+use reqwest::StatusCode;
mod create;
mod debug;
@@ -26,6 +29,63 @@ mod render_layer;
mod settings;
mod workflow;
+fn load_bundled_signatures() {
+ let global_bn_settings = Settings::new();
+ let plugin_settings = PluginSettings::from_settings(&global_bn_settings);
+ // We want to load all the bundled directories into the container cache.
+ let background_task = BackgroundTask::new("Loading WARP files...", false);
+ let start = Instant::now();
+ if plugin_settings.load_bundled_files {
+ let core_disk_container = DiskContainer::new_from_dir(core_signature_dir());
+ log::debug!("{:#?}", core_disk_container);
+ add_cached_container(core_disk_container);
+ }
+ if plugin_settings.load_user_files {
+ let user_disk_container = DiskContainer::new_from_dir(user_signature_dir());
+ log::debug!("{:#?}", user_disk_container);
+ add_cached_container(user_disk_container);
+ }
+ log::info!("Loading bundled files took {:?}", start.elapsed());
+ background_task.finish();
+}
+
+fn load_network_container() {
+ let global_bn_settings = Settings::new();
+ let plugin_settings = PluginSettings::from_settings(&global_bn_settings);
+ let background_task = BackgroundTask::new("Initializing WARP server...", false);
+ let start = Instant::now();
+ if plugin_settings.enable_server {
+ let server_url = plugin_settings.server_url.clone();
+ let server_api_key = plugin_settings.server_api_key.clone();
+ let https_proxy_str = global_bn_settings.get_string("network.httpsProxy");
+ let https_proxy = if https_proxy_str.is_empty() {
+ None
+ } else {
+ Some(https_proxy_str)
+ };
+ match NetworkClient::new(server_url.clone(), server_api_key, https_proxy) {
+ Ok(network_client) => {
+ // Before constructing the container, let's make sure that the server is OK.
+ if let Ok(StatusCode::OK) = network_client.status() {
+ let network_container = NetworkContainer::new(network_client);
+ log::debug!("{:#?}", network_container);
+ add_cached_container(network_container);
+ } else {
+ log::error!(
+ "Server '{}' is not reachable, disabling container...",
+ server_url
+ );
+ }
+ }
+ Err(e) => {
+ log::error!("Failed to add networked container: {}", e);
+ }
+ }
+ }
+ log::debug!("Initializing warp server took {:?}", start.elapsed());
+ background_task.finish();
+}
+
#[no_mangle]
#[allow(non_snake_case)]
pub extern "C" fn CorePluginInit() -> bool {
@@ -44,22 +104,21 @@ pub extern "C" fn CorePluginInit() -> bool {
workflow::insert_workflow();
- let plugin_settings = PluginSettings::from_settings(&global_bn_settings);
- // We want to load all the bundled directories into the container cache.
- let background_task = BackgroundTask::new("Loading WARP files...", false);
- let start = Instant::now();
- if plugin_settings.load_bundled_files {
- let core_disk_container = DiskContainer::new_from_dir(core_signature_dir());
- log::debug!("{:#?}", core_disk_container);
- add_cached_container(core_disk_container);
+ // TODO: Make the retrieval of containers wait on this to be done.
+ // TODO: We could also have a mechanism for lazily loading the files using the chunk header target.
+ // Loading bundled signatures might take a few hundred milliseconds.
+ if is_ui_enabled() {
+ std::thread::spawn(|| {
+ load_bundled_signatures();
+ load_network_container();
+ });
+ } else {
+ load_bundled_signatures();
+ std::thread::spawn(|| {
+ // Dependence on this is likely to not matter in headless, so we throw it on another thread.
+ load_network_container();
+ });
}
- if plugin_settings.load_user_files {
- let user_disk_container = DiskContainer::new_from_dir(user_signature_dir());
- log::debug!("{:#?}", user_disk_container);
- add_cached_container(user_disk_container);
- }
- log::info!("Loading bundled files took {:?}", start.elapsed());
- background_task.finish();
register_command(
"WARP\\Run Matcher",
diff --git a/plugins/warp/src/plugin/ffi/container.rs b/plugins/warp/src/plugin/ffi/container.rs
index 21ad4dc0..4de6bab3 100644
--- a/plugins/warp/src/plugin/ffi/container.rs
+++ b/plugins/warp/src/plugin/ffi/container.rs
@@ -36,6 +36,27 @@ pub unsafe extern "C" fn BNWARPContainerGetName(container: *mut BNWARPContainer)
}
#[no_mangle]
+pub unsafe extern "C" fn BNWARPContainerFetchFunctions(
+ container: *mut BNWARPContainer,
+ target: *mut BNWARPTarget,
+ guids: *const BNWARPFunctionGUID,
+ count: usize,
+) {
+ let arc_container = ManuallyDrop::new(Arc::from_raw(container));
+ let Ok(mut container) = arc_container.write() else {
+ return;
+ };
+
+ let target = unsafe { ManuallyDrop::new(Arc::from_raw(target)) };
+
+ let guids = unsafe { std::slice::from_raw_parts(guids, count) };
+
+ if let Err(e) = container.fetch_functions(&target, guids) {
+ log::error!("Failed to fetch functions: {}", e);
+ }
+}
+
+#[no_mangle]
pub unsafe extern "C" fn BNWARPContainerGetSources(
container: *mut BNWARPContainer,
count: *mut usize,
diff --git a/plugins/warp/src/plugin/settings.rs b/plugins/warp/src/plugin/settings.rs
index 6cd2e0cc..489deb23 100644
--- a/plugins/warp/src/plugin/settings.rs
+++ b/plugins/warp/src/plugin/settings.rs
@@ -19,7 +19,7 @@ pub struct PluginSettings {
/// The API key to use for the selected WARP server, if not specified, you will be unable to push data and may be rate-limited.
///
/// This is set to [PluginSettings::SERVER_API_KEY_DEFAULT] by default.
- pub server_api_key: String,
+ pub server_api_key: Option<String>,
/// Whether to allow networked WARP requests. Turning this off will not disable local WARP functionality.
///
/// This is set to [PluginSettings::ENABLE_SERVER_DEFAULT] by default.
@@ -33,9 +33,9 @@ impl PluginSettings {
pub const LOAD_USER_FILES_SETTING: &'static str = "analysis.warp.loadUserFiles";
pub const SERVER_URL_DEFAULT: &'static str = "https://warp.binary.ninja";
pub const SERVER_URL_SETTING: &'static str = "analysis.warp.serverUrl";
- pub const SERVER_API_KEY_DEFAULT: &'static str = "";
+ pub const SERVER_API_KEY_DEFAULT: Option<String> = None;
pub const SERVER_API_KEY_SETTING: &'static str = "analysis.warp.serverApiKey";
- pub const ENABLE_SERVER_DEFAULT: bool = true;
+ pub const ENABLE_SERVER_DEFAULT: bool = false;
pub const ENABLE_SERVER_SETTING: &'static str = "network.enableWARP";
pub fn register(bn_settings: &mut BNSettings) {
@@ -44,7 +44,8 @@ impl PluginSettings {
"type" : "boolean",
"default" : Self::LOAD_BUNDLED_FILES_DEFAULT,
"description" : "Whether to load bundled WARP files on startup. Turn this off if you want to manually load them.",
- "ignore" : ["SettingsProjectScope", "SettingsResourceScope"]
+ "ignore" : ["SettingsProjectScope", "SettingsResourceScope"],
+ "requiresRestart" : true
});
bn_settings.register_setting_json(
Self::LOAD_BUNDLED_FILES_SETTING,
@@ -55,7 +56,8 @@ impl PluginSettings {
"type" : "boolean",
"default" : Self::LOAD_USER_FILES_DEFAULT,
"description" : "Whether to load user WARP files on startup. Turn this off if you want to manually load them.",
- "ignore" : ["SettingsProjectScope", "SettingsResourceScope"]
+ "ignore" : ["SettingsProjectScope", "SettingsResourceScope"],
+ "requiresRestart" : true
});
bn_settings.register_setting_json(
Self::LOAD_USER_FILES_SETTING,
@@ -66,7 +68,8 @@ impl PluginSettings {
"type" : "string",
"default" : Self::SERVER_URL_DEFAULT,
"description" : "The WARP server to use.",
- "ignore" : ["SettingsProjectScope", "SettingsResourceScope"]
+ "ignore" : ["SettingsProjectScope", "SettingsResourceScope"],
+ "requiresRestart" : true
});
bn_settings.register_setting_json(Self::SERVER_URL_SETTING, &server_url_prop.to_string());
let server_api_key_prop = json!({
@@ -75,7 +78,8 @@ impl PluginSettings {
"default" : Self::SERVER_API_KEY_DEFAULT,
"description" : "The API key to use for the selected WARP server, if not specified you will be unable to push data, and may be rate limited.",
"ignore" : ["SettingsProjectScope", "SettingsResourceScope"],
- "hidden": true
+ "hidden": true,
+ "requiresRestart" : true
});
bn_settings.register_setting_json(
Self::SERVER_API_KEY_SETTING,
@@ -86,7 +90,8 @@ impl PluginSettings {
"type" : "boolean",
"default" : Self::ENABLE_SERVER_DEFAULT,
"description" : "Whether or not to allow networked WARP requests. Turning this off will not disable local WARP functionality.",
- "ignore" : ["SettingsProjectScope", "SettingsResourceScope"]
+ "ignore" : ["SettingsProjectScope", "SettingsResourceScope"],
+ "requiresRestart" : true
});
bn_settings.register_setting_json(
Self::ENABLE_SERVER_SETTING,
@@ -107,7 +112,10 @@ impl PluginSettings {
settings.server_url = bn_settings.get_string(Self::SERVER_URL_SETTING);
}
if bn_settings.contains(Self::SERVER_API_KEY_SETTING) {
- settings.server_url = bn_settings.get_string(Self::SERVER_API_KEY_SETTING);
+ let server_api_key_str = bn_settings.get_string(Self::SERVER_API_KEY_SETTING);
+ if !server_api_key_str.is_empty() {
+ settings.server_api_key = Some(server_api_key_str);
+ }
}
if bn_settings.contains(Self::ENABLE_SERVER_SETTING) {
settings.enable_server = bn_settings.get_bool(Self::ENABLE_SERVER_SETTING);
@@ -122,7 +130,7 @@ impl Default for PluginSettings {
load_bundled_files: PluginSettings::LOAD_BUNDLED_FILES_DEFAULT,
load_user_files: PluginSettings::LOAD_USER_FILES_DEFAULT,
server_url: PluginSettings::SERVER_URL_DEFAULT.to_string(),
- server_api_key: PluginSettings::SERVER_API_KEY_DEFAULT.to_string(),
+ server_api_key: PluginSettings::SERVER_API_KEY_DEFAULT,
enable_server: PluginSettings::ENABLE_SERVER_DEFAULT,
}
}
diff --git a/plugins/warp/src/plugin/workflow.rs b/plugins/warp/src/plugin/workflow.rs
index 2d76222c..c2f82bd9 100644
--- a/plugins/warp/src/plugin/workflow.rs
+++ b/plugins/warp/src/plugin/workflow.rs
@@ -60,6 +60,15 @@ impl Command for RunMatcher {
fn action(&self, view: &BinaryView) {
let view = view.to_owned();
std::thread::spawn(move || {
+ // For embedded targets the user may not have set the sections up.
+ // Alert the user if we have no actual regions (+1 comes from the synthetic section).
+ let regions = relocatable_regions(&view);
+ if regions.len() <= 1 && view.memory_map().is_activated() {
+ log::warn!(
+ "No relocatable regions found, for best results please define sections for the binary!"
+ );
+ }
+
run_matcher(&view);
});
}
@@ -75,14 +84,6 @@ pub fn run_matcher(view: &BinaryView) {
let _ = get_warp_tag_type(view);
view.file().forget_undo_actions(&undo_id);
- // Alert the user if we have no actual regions (one comes from the synthetic section).
- let regions = relocatable_regions(view);
- if regions.len() <= 1 && view.memory_map().is_activated() {
- log::warn!(
- "No relocatable regions found, for best results please define sections for the binary!"
- );
- }
-
// Then we want to actually find matching functions.
let background_task = BackgroundTask::new("Matching on WARP functions...", true);
let start = Instant::now();
diff --git a/plugins/warp/ui/matches.cpp b/plugins/warp/ui/matches.cpp
index fad729d6..ed77085a 100644
--- a/plugins/warp/ui/matches.cpp
+++ b/plugins/warp/ui/matches.cpp
@@ -5,6 +5,7 @@
#include <QClipboard>
#include <QFormLayout>
+#include <thread>
#include "theme.h"
#include "warp.h"
@@ -15,6 +16,8 @@ WarpCurrentFunctionWidget::WarpCurrentFunctionWidget(FunctionRef current)
// NOTE: Might be nullptr if the no selected function.
m_current = current;
+ m_logger = new BinaryNinja::Logger("WARP");
+
// Create the QT stuff
QGridLayout *layout = new QGridLayout(this);
layout->setContentsMargins(2, 2, 2, 2);
@@ -110,6 +113,26 @@ void WarpCurrentFunctionWidget::SetCurrentFunction(FunctionRef current)
return;
m_current = current;
m_infoWidget->SetAnalysisFunction(m_current);
+
+ if (current)
+ {
+ // Add the function to the processing list only if we have no already done so.
+ // If a user goes to a function, then navigates away, they do not want to
+ // have us try and send a network request!
+ {
+ std::lock_guard<std::mutex> lock(m_requestMutex);
+ uint64_t funcStart = current->GetStart();
+ if (m_processedFunctions.find(funcStart) == m_processedFunctions.end()) {
+ m_pendingRequests.push_back(current);
+ }
+ }
+ if (!m_requestInProgress.exchange(true)) {
+ BinaryNinja::WorkerPriorityEnqueue([this]() {
+ ProcessPendingFetchRequests();
+ });
+ }
+ }
+
UpdateMatches();
}
@@ -162,3 +185,51 @@ void WarpCurrentFunctionWidget::UpdateMatches()
m_tableWidget->SetFunctions(matches);
}
+
+void WarpCurrentFunctionWidget::ProcessPendingFetchRequests()
+{
+ std::vector<FunctionRef> requests;
+ {
+ std::lock_guard<std::mutex> lock(m_requestMutex);
+ requests = std::move(m_pendingRequests);
+ m_pendingRequests.clear();
+ }
+
+ if (requests.empty()) {
+ m_requestInProgress = false;
+ return;
+ }
+
+ auto start_time = std::chrono::high_resolution_clock::now();
+
+ std::vector<Warp::FunctionGUID> guids;
+ Warp::Ref<Warp::Target> target;
+ for (const auto& func : requests) {
+ // TODO: Need to send multiple requests if there is multiple targets.
+ if (!target)
+ target = Warp::Target::FromPlatform(*func->GetPlatform());
+ if (const auto guid = Warp::GetAnalysisFunctionGUID(*func); guid.has_value())
+ guids.push_back(guid.value());
+ }
+
+ // Actually fetch the data!
+ if (!guids.empty())
+ for (const auto &container: Warp::Container::All())
+ container->FetchFunctions(*target, guids);
+
+ {
+ std::lock_guard<std::mutex> lock(m_requestMutex);
+ for (const auto& func : requests) {
+ m_processedFunctions.insert(func->GetStart());
+ }
+ }
+
+ // TODO: Update the matches, make sure there was stuff added first lol.
+ // TODO: UpdateMatches();
+
+ const auto end_time = std::chrono::high_resolution_clock::now();
+ const std::chrono::duration<double> elapsed_time = end_time - start_time;
+ m_logger->LogDebug("ProcessPendingRequests took %f seconds", elapsed_time.count());
+
+ m_requestInProgress = false;
+}
diff --git a/plugins/warp/ui/matches.h b/plugins/warp/ui/matches.h
index 4021f10c..251dcb89 100644
--- a/plugins/warp/ui/matches.h
+++ b/plugins/warp/ui/matches.h
@@ -16,6 +16,13 @@ class WarpCurrentFunctionWidget : public QWidget
WarpFunctionTableWidget *m_tableWidget;
WarpFunctionInfoWidget *m_infoWidget;
+ LoggerRef m_logger;
+
+ std::mutex m_requestMutex;
+ std::vector<FunctionRef> m_pendingRequests;
+ std::atomic<bool> m_requestInProgress {false};
+ std::unordered_set<uint64_t> m_processedFunctions;
+
public:
explicit WarpCurrentFunctionWidget(FunctionRef current);
@@ -26,4 +33,6 @@ public:
FunctionRef GetCurrentFunction() { return m_current; };
void UpdateMatches();
+
+ void ProcessPendingFetchRequests();
};
diff --git a/plugins/warp/ui/plugin.cpp b/plugins/warp/ui/plugin.cpp
index 69aaaf34..923a79b3 100644
--- a/plugins/warp/ui/plugin.cpp
+++ b/plugins/warp/ui/plugin.cpp
@@ -170,6 +170,9 @@ void WarpSidebarWidget::notifyViewChanged(ViewFrame *view)
void WarpSidebarWidget::notifyViewLocationChanged(View *view, const ViewLocation &location)
{
+ // Warp sidebar really should only update if it is visible, otherwise its a waste of cycles.
+ if (!this->isVisible())
+ return;
auto function = location.getFunction();
// TODO: Only update if the function exists?
// NOTE: The function called will exit early if it is the same function.