diff options
Diffstat (limited to 'plugins')
| -rw-r--r-- | plugins/warp/Cargo.toml | 1 | ||||
| -rw-r--r-- | plugins/warp/api/python/_warpcore.py | 23 | ||||
| -rw-r--r-- | plugins/warp/api/python/warp.py | 7 | ||||
| -rw-r--r-- | plugins/warp/api/warp.cpp | 10 | ||||
| -rw-r--r-- | plugins/warp/api/warp.h | 2 | ||||
| -rw-r--r-- | plugins/warp/api/warpcore.h | 2 | ||||
| -rw-r--r-- | plugins/warp/src/container.rs | 13 | ||||
| -rw-r--r-- | plugins/warp/src/container/network.rs | 332 | ||||
| -rw-r--r-- | plugins/warp/src/container/network/client.rs | 228 | ||||
| -rw-r--r-- | plugins/warp/src/plugin.rs | 89 | ||||
| -rw-r--r-- | plugins/warp/src/plugin/ffi/container.rs | 21 | ||||
| -rw-r--r-- | plugins/warp/src/plugin/settings.rs | 28 | ||||
| -rw-r--r-- | plugins/warp/src/plugin/workflow.rs | 17 | ||||
| -rw-r--r-- | plugins/warp/ui/matches.cpp | 71 | ||||
| -rw-r--r-- | plugins/warp/ui/matches.h | 9 | ||||
| -rw-r--r-- | plugins/warp/ui/plugin.cpp | 3 |
16 files changed, 812 insertions, 44 deletions
diff --git a/plugins/warp/Cargo.toml b/plugins/warp/Cargo.toml index afebf577..761ac961 100644 --- a/plugins/warp/Cargo.toml +++ b/plugins/warp/Cargo.toml @@ -24,6 +24,7 @@ thiserror = "2.0" ar = { git = "https://github.com/mdsteele/rust-ar" } tempdir = "0.3.7" regex = "1.11" +reqwest = { version = "0.12", features = ["blocking", "json", "multipart"] } # For reports minijinja = "2.10.2" diff --git a/plugins/warp/api/python/_warpcore.py b/plugins/warp/api/python/_warpcore.py index 740072ab..da262f18 100644 --- a/plugins/warp/api/python/_warpcore.py +++ b/plugins/warp/api/python/_warpcore.py @@ -205,6 +205,29 @@ def BNWARPContainerCommitSource( # ------------------------------------------------------- +# _BNWARPContainerFetchFunctions + +_BNWARPContainerFetchFunctions = core.BNWARPContainerFetchFunctions +_BNWARPContainerFetchFunctions.restype = None +_BNWARPContainerFetchFunctions.argtypes = [ + ctypes.POINTER(BNWARPContainer), + ctypes.POINTER(BNWARPTarget), + ctypes.POINTER(BNWARPTypeGUID), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNWARPContainerFetchFunctions( + container: ctypes.POINTER(BNWARPContainer), + target: ctypes.POINTER(BNWARPTarget), + guids: ctypes.POINTER(BNWARPTypeGUID), + count: int + ) -> None: + return _BNWARPContainerFetchFunctions(container, target, guids, count) + + +# ------------------------------------------------------- # _BNWARPContainerGetFunctionsWithGUID _BNWARPContainerGetFunctionsWithGUID = core.BNWARPContainerGetFunctionsWithGUID diff --git a/plugins/warp/api/python/warp.py b/plugins/warp/api/python/warp.py index c761c323..c7486b8c 100644 --- a/plugins/warp/api/python/warp.py +++ b/plugins/warp/api/python/warp.py @@ -305,6 +305,13 @@ class WarpContainer(metaclass=_WarpContainerMetaclass): core_guids[i] = guids[i].uuid return warpcore.BNWARPContainerRemoveTypes(self.handle, source.uuid, core_guids, count) + def fetch_functions(self, target: WarpTarget, guids: List[FunctionGUID]): + count = len(guids) + core_guids = (warpcore.BNWARPFunctionGUID * count)() + for i in range(count): + core_guids[i] = guids[i].uuid + warpcore.BNWARPContainerFetchFunctions(self.handle, target.handle, core_guids, count) + def get_sources_with_function_guid(self, target: WarpTarget, guid: FunctionGUID) -> List[Source]: count = ctypes.c_size_t() sources = warpcore.BNWARPContainerGetSourcesWithFunctionGUID(self.handle, target.handle, guid.uuid, count) diff --git a/plugins/warp/api/warp.cpp b/plugins/warp/api/warp.cpp index 147add8e..52c7ea9d 100644 --- a/plugins/warp/api/warp.cpp +++ b/plugins/warp/api/warp.cpp @@ -249,6 +249,16 @@ bool Container::RemoveTypes(const Source &source, const std::vector<TypeGUID> &g return result; } +void Container::FetchFunctions(const Target &target, const std::vector<FunctionGUID> &guids) const +{ + size_t count = guids.size(); + BNWARPFunctionGUID *apiGuids = new BNWARPFunctionGUID[count]; + for (size_t i = 0; i < count; i++) + apiGuids[i] = *guids[i].Raw(); + BNWARPContainerFetchFunctions(m_object, target.m_object, apiGuids, count); + delete[] apiGuids; +} + std::vector<Source> Container::GetSourcesWithFunctionGUID(const Target& target, const FunctionGUID &guid) const { size_t count; diff --git a/plugins/warp/api/warp.h b/plugins/warp/api/warp.h index b57a0c7b..807ba998 100644 --- a/plugins/warp/api/warp.h +++ b/plugins/warp/api/warp.h @@ -354,6 +354,8 @@ namespace Warp { bool RemoveTypes(const Source &source, const std::vector<TypeGUID> &guids) const; + void FetchFunctions(const Target& target, const std::vector<FunctionGUID> &guids) const; + std::vector<Source> GetSourcesWithFunctionGUID(const Target& target, const FunctionGUID &guid) const; std::vector<Source> GetSourcesWithTypeGUID(const TypeGUID &guid) const; diff --git a/plugins/warp/api/warpcore.h b/plugins/warp/api/warpcore.h index 09190126..20a59b7b 100644 --- a/plugins/warp/api/warpcore.h +++ b/plugins/warp/api/warpcore.h @@ -108,6 +108,8 @@ extern "C" WARP_FFI_API bool BNWARPContainerRemoveFunctions(BNWARPContainer* container, const BNWARPTarget* target, const BNWARPSource* source, BNWARPFunction** functions, size_t count); WARP_FFI_API bool BNWARPContainerRemoveTypes(BNWARPContainer* container, const BNWARPSource* source, BNWARPTypeGUID* types, size_t count); + WARP_FFI_API void BNWARPContainerFetchFunctions(BNWARPContainer* container, BNWARPTarget* target, const BNWARPTypeGUID* guids, size_t count); + WARP_FFI_API BNWARPSource* BNWARPContainerGetSourcesWithFunctionGUID(BNWARPContainer* container, const BNWARPTarget* target, const BNWARPFunctionGUID* guid, size_t* count); WARP_FFI_API BNWARPSource* BNWARPContainerGetSourcesWithTypeGUID(BNWARPContainer* container, const BNWARPTypeGUID* guid, size_t* count); WARP_FFI_API BNWARPFunction** BNWARPContainerGetFunctionsWithGUID(BNWARPContainer* container, const BNWARPTarget* target, const BNWARPSource* source, const BNWARPFunctionGUID* guid, size_t* count); diff --git a/plugins/warp/src/container.rs b/plugins/warp/src/container.rs index 8d720021..4cabc60e 100644 --- a/plugins/warp/src/container.rs +++ b/plugins/warp/src/container.rs @@ -208,6 +208,19 @@ pub trait Container: Send + Sync + Display + Debug { functions: &[Function], ) -> ContainerResult<()>; + /// Fetches WARP information for the associated functions. + /// + /// Typically, a container that resides only in memory has nothing to fetch, so the default implementation + /// will do nothing. This function is blocking, so assume it will take a few seconds for a container + /// that intends to fetch over the network. + fn fetch_functions( + &mut self, + _target: &Target, + _functions: &[FunctionGUID], + ) -> ContainerResult<()> { + Ok(()) + } + /// Get the sources that contain a type with the given [`TypeGUID`]. fn sources_with_type_guid(&self, guid: &TypeGUID) -> ContainerResult<Vec<SourceId>>; diff --git a/plugins/warp/src/container/network.rs b/plugins/warp/src/container/network.rs index ffbe6108..2a2d7c65 100644 --- a/plugins/warp/src/container/network.rs +++ b/plugins/warp/src/container/network.rs @@ -1,13 +1,323 @@ -pub struct NetworkContainer {} +use crate::container::disk::DiskContainer; +use crate::container::{Container, ContainerError, ContainerResult, SourceId, SourcePath}; +use std::collections::HashMap; +use std::fmt::{Debug, Display, Formatter}; +use warp::chunk::{Chunk, ChunkKind, CompressionType}; +use warp::r#type::guid::TypeGUID; +use warp::r#type::{ComputedType, Type}; +use warp::signature::chunk::SignatureChunk; +use warp::signature::function::{Function, FunctionGUID}; +use warp::target::Target; +use warp::{WarpFile, WarpFileHeader}; -// TODO: The container is populated as the user is navigating a binary -// TODO: We need to have a few helper functions here to post and pull -// TODO: Then in the interface we operate off the network cache -// TODO: The network cache could just be a disk container? Or disk container sources? -// TODO: We should also store the cache on the filesystem for a certain time, will need to timestamp -// TODO: When we commit we need to actually POST i believe. -// TODO: There needs to be a setting that adjusts the sweep size of functions at the cursor. -// TODO: Probably need a callback or something to tell the network containers to refresh from the network. -// TODO: The network container should never instantiate itself, unless its gurenteed to not have any data in it? +pub mod client; -// TODO: Need to PUSH chunks and PULL chunks +pub use client::NetworkClient; + +/// This is the id on the server for the [`Target`], we can get it via [`NetworkClient::query_target_id`]. +pub type NetworkTargetId = i32; + +pub struct NetworkContainer { + client: NetworkClient, + /// This is the store that the interface will write to; then we have special functions for pulling + /// and pushing to the network source. + cache: DiskContainer, + /// Populated when targets are queried. + known_targets: HashMap<Target, Option<NetworkTargetId>>, + /// Populated with function sources are queried. + known_function_sources: HashMap<FunctionGUID, Vec<SourceId>>, + /// Populated when user adds function, this is used for writing back to the server. + added_chunks: HashMap<SourceId, Vec<Chunk<'static>>>, +} + +impl NetworkContainer { + pub fn new(client: NetworkClient) -> Self { + Self { + cache: DiskContainer::new("Network Container".to_string(), HashMap::new()), + client, + known_targets: HashMap::new(), + known_function_sources: HashMap::new(), + added_chunks: HashMap::new(), + } + } + + /// Gets the network id for the `target`, this will be used in later function queries. + /// + /// **This is blocking** + /// + /// # Caching policy + /// + /// The [`NetworkTargetId`] is unique and immutable, so they will be persisted indefinitely. + pub fn get_target_id(&mut self, target: &Target) -> Option<NetworkTargetId> { + // It's highly probable we have previously queried the target, check that first. + if let Some(target_id) = self.known_targets.get(target) { + return target_id.clone(); + } + + let target_id = self.client.query_target_id(target); + // Keep the target id so the next lookup is free. + self.known_targets.insert(target.clone(), target_id); + target_id + } + + /// Pulls sources for the set of unseen function guids. + /// + /// **This is blocking** + /// + /// # Caching policy + /// + /// When we get the source, we store the results indefinitely in the container; this is fine + /// for now as the requests for functions come at the request of some user interaction. Any guid + /// with no sources will still be cached. + pub fn get_unseen_functions_source( + &mut self, + target: Option<&Target>, + guids: &[FunctionGUID], + ) -> HashMap<SourceId, Vec<FunctionGUID>> { + let Some(target_id) = target.and_then(|t| self.get_target_id(t)) else { + log::debug!("Cannot query functions source without a target, skipping..."); + return HashMap::new(); + }; + + // Split guids into known and unknown + let (_known, unknown): (Vec<_>, Vec<_>) = guids + .into_iter() + .cloned() + .partition(|guid| self.known_function_sources.contains_key(guid)); + + let mut result: HashMap<SourceId, Vec<FunctionGUID>> = HashMap::new(); + // Only query server for unknown guids if we have any. + if !unknown.is_empty() { + if let Some(queried_results) = self + .client + .query_functions_source(Some(target_id), &unknown) + { + // Cache the new results, this means we will not try and contact the server for that guids source. + // NOTE: Here we do not just simply list the queried results because we also + // want to cache function guids which have no source, this is important so that we never + // attempt to contact the server for that guid. + for guid in &unknown { + let sources = queried_results + .keys() + .filter(|source_id| queried_results[source_id].contains(guid)) + .cloned() + .collect(); + self.known_function_sources.insert(*guid, sources); + } + + for (source_id, guids) in queried_results { + result.entry(source_id).or_default().extend(guids); + } + } + } + + result + } + + /// Pulls function metadata from the server and adds it into the container cache. + /// + /// **This is blocking** + /// + /// # Caching policy + /// + /// Every request we store the returned objects on disk, this means that users will first + /// query against the disk objects, then the server. This also means we need to cache functions f + /// or which we have not received any functions for, as otherwise we would keep trying to query it. + pub fn pull_functions( + &mut self, + target: &Target, + source: &SourceId, + functions: &[FunctionGUID], + ) { + let target_id = self.get_target_id(target); + if let Some(file) = self + .client + .query_functions(target_id, Some(*source), functions) + { + log::debug!("Got {} chunks from server", file.chunks.len()); + for chunk in &file.chunks { + match &chunk.kind { + ChunkKind::Signature(sc) => { + let functions: Vec<_> = sc.functions().collect(); + match self.cache.add_functions(target, source, &functions) { + Ok(_) => log::debug!( + "Added {} functions into cached source '{}'", + functions.len(), + source + ), + Err(err) => log::error!( + "Failed to add {} function into cached source '{}': {}", + functions.len(), + source, + err + ), + } + } + // TODO; Probably want to pull type in with this. + ChunkKind::Type(_) => {} + } + } + } + } + + /// Push a file to the network source. + /// + /// **This is blocking** + pub fn push_file(&mut self, source_id: SourceId, file: &WarpFile) { + self.client.push_file(source_id, file); + } +} + +impl Container for NetworkContainer { + fn sources(&self) -> ContainerResult<Vec<SourceId>> { + self.cache.sources() + } + + fn add_source(&mut self, path: SourcePath) -> ContainerResult<SourceId> { + // TODO: How do we want to let users create new sources? + log::error!("NetworkContainer::add_source not allowed"); + Err(ContainerError::CannotCreateSource(path)) + } + + fn commit_source(&mut self, source: &SourceId) -> ContainerResult<bool> { + let chunks = self + .added_chunks + .remove(source) + .ok_or(ContainerError::SourceNotFound(source.clone()))?; + let file = WarpFile::new(WarpFileHeader::new(), chunks); + self.push_file(*source, &file); + Ok(true) + } + + fn is_source_writable(&self, source: &SourceId) -> ContainerResult<bool> { + // TODO: This is retrievable from /users/me/sources we will grab it when connecting. + log::error!("NetworkContainer::is_source_writable not allowed"); + Err(ContainerError::SourceNotWritable(source.clone())) + } + + fn is_source_uncommitted(&self, source: &SourceId) -> ContainerResult<bool> { + Ok(self.added_chunks.contains_key(source)) + } + + fn source_path(&self, source: &SourceId) -> ContainerResult<SourcePath> { + self.cache.source_path(source) + } + + fn add_computed_types( + &mut self, + source: &SourceId, + types: &[ComputedType], + ) -> ContainerResult<()> { + self.cache.add_computed_types(source, types) + } + + fn remove_types(&mut self, source: &SourceId, guids: &[TypeGUID]) -> ContainerResult<()> { + self.cache.remove_types(source, guids) + } + + fn add_functions( + &mut self, + target: &Target, + source: &SourceId, + functions: &[Function], + ) -> ContainerResult<()> { + let signature_chunk = SignatureChunk::new(functions).ok_or( + ContainerError::CorruptedData("signature chunk failed to validate"), + )?; + let chunk = Chunk::new_with_target( + ChunkKind::Signature(signature_chunk), + CompressionType::None, + target.clone(), + ); + self.added_chunks.entry(*source).or_default().push(chunk); + Ok(()) + } + + fn remove_functions( + &mut self, + target: &Target, + source: &SourceId, + functions: &[Function], + ) -> ContainerResult<()> { + // TODO: Wont persist, need to add remote removal. + self.cache.remove_functions(target, source, functions) + } + + fn fetch_functions( + &mut self, + target: &Target, + functions: &[FunctionGUID], + ) -> ContainerResult<()> { + // NOTE: Blocking request to get the mapped function sources. + let mapped_unseen_functions = self.get_unseen_functions_source(Some(&target), functions); + + // Actually get the function data for the unseen guids, we really only want to do this once per + // session, anymore, and this is annoying! + for (source, unseen_guids) in mapped_unseen_functions { + // NOTE: Blocking request to get the function data in the container cache. + self.pull_functions(&target, &source, &unseen_guids); + } + + Ok(()) + } + + fn sources_with_type_guid(&self, guid: &TypeGUID) -> ContainerResult<Vec<SourceId>> { + self.cache.sources_with_type_guid(guid) + } + + fn sources_with_type_guids( + &self, + guids: &[TypeGUID], + ) -> ContainerResult<HashMap<TypeGUID, Vec<SourceId>>> { + self.cache.sources_with_type_guids(guids) + } + + fn type_guids_with_name( + &self, + source: &SourceId, + name: &str, + ) -> ContainerResult<Vec<TypeGUID>> { + self.cache.type_guids_with_name(source, name) + } + + fn type_with_guid(&self, source: &SourceId, guid: &TypeGUID) -> ContainerResult<Option<Type>> { + self.cache.type_with_guid(source, guid) + } + + fn sources_with_function_guid( + &self, + target: &Target, + guid: &FunctionGUID, + ) -> ContainerResult<Vec<SourceId>> { + self.cache.sources_with_function_guid(target, guid) + } + + fn sources_with_function_guids( + &self, + target: &Target, + guids: &[FunctionGUID], + ) -> ContainerResult<HashMap<FunctionGUID, Vec<SourceId>>> { + self.cache.sources_with_function_guids(target, guids) + } + + fn functions_with_guid( + &self, + target: &Target, + source: &SourceId, + guid: &FunctionGUID, + ) -> ContainerResult<Vec<Function>> { + self.cache.functions_with_guid(target, source, guid) + } +} + +impl Debug for NetworkContainer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NetworkContainer").finish() + } +} + +impl Display for NetworkContainer { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("NetworkContainer").finish() + } +} diff --git a/plugins/warp/src/container/network/client.rs b/plugins/warp/src/container/network/client.rs new file mode 100644 index 00000000..f77f1118 --- /dev/null +++ b/plugins/warp/src/container/network/client.rs @@ -0,0 +1,228 @@ +use crate::container::network::NetworkTargetId; +use crate::container::SourceId; +use reqwest::blocking::Client; +use reqwest::header::{HeaderMap, HeaderValue, AUTHORIZATION}; +use reqwest::StatusCode; +use serde_json::json; +use std::collections::HashMap; +use std::str::FromStr; +use warp::signature::function::FunctionGUID; +use warp::target::Target; +use warp::WarpFile; + +/// Responsible for sending and receiving data from the server. +/// +/// NOTE: **All requests are blocking**. +#[derive(Clone, Debug)] +pub struct NetworkClient { + client: Client, + server_url: String, +} + +impl NetworkClient { + pub fn new( + server_url: String, + server_token: Option<String>, + https_proxy: Option<String>, + ) -> reqwest::Result<Self> { + let version_info = binaryninja::version_info(); + // TODO: IIRC we had a user agent format already for some other thing. + let client_agent = format!( + "Binary Ninja/{}.{}.{}", + version_info.major, version_info.minor, version_info.build + ); + // TODO: This might want to be kept for the request header? + let mut headers = HeaderMap::new(); + if let Some(token) = &server_token { + headers.insert( + AUTHORIZATION, + HeaderValue::from_str(&format!("Bearer {}", token)).unwrap(), + ); + } + // TODO: Configurable timeout? + let mut client_builder = Client::builder() + .connect_timeout(std::time::Duration::from_secs(10)) + .default_headers(headers) + .user_agent(client_agent); + if let Some(https_proxy) = https_proxy { + client_builder = client_builder.proxy(reqwest::Proxy::all(&https_proxy)?); + } + Ok(Self { + client: client_builder.build()?, + server_url, + }) + } + + /// Check to see the status of the server. + /// + /// This is useful if you want to fail early and prevent constructing a network container to a + /// server that is unresponsive. + /// + /// Route: `api/v1/status` + pub fn status(&self) -> reqwest::Result<StatusCode> { + let status_url = format!("{}/api/v1/status", self.server_url); + let resp = self.client.get(&status_url).send()?; + Ok(resp.status()) + } + + /// Query the [`NetworkTargetId`] for the given [`Target`]. + /// + /// NOTE: **THIS IS BLOCKING** + /// + /// Route: `api/v1/targets/query` (TODO: Comment about the query) + pub fn query_target_id(&self, target: &Target) -> Option<NetworkTargetId> { + let query_target_url = format!("{}/api/v1/targets/query", self.server_url); + + let mut query = HashMap::new(); + if let Some(platform) = &target.platform { + query.insert("platform", platform); + } + if let Some(architecture) = &target.architecture { + query.insert("architecture", architecture); + } + + // NOTE: This is blocking. + let target_id: NetworkTargetId = self + .client + .get(query_target_url) + .query(&query) + .send() + .ok()? + .json::<NetworkTargetId>() + .ok()?; + + Some(target_id) + } + + fn query_functions_body( + target: Option<NetworkTargetId>, + source: Option<SourceId>, + guids: &[FunctionGUID], + ) -> serde_json::Value { + let guids_str: Vec<String> = guids.iter().map(|g| g.to_string()).collect(); + // TODO: The limit here needs to be somewhat flexible. But 1000 will do for now. + let mut body = json!({ + "format": "flatbuffer", + "guids": guids_str, + "limit": 1000 + }); + if let Some(target_id) = target { + body["target_id"] = json!(target_id); + } + if let Some(source_id) = source { + body["source_id"] = json!(source_id.to_string()); + } + body + } + + /// Query the functions, returning the warp file response containing the entries. + /// + /// NOTE: **THIS IS BLOCKING** + /// + /// Route: `api/v1/functions/query` (TODO: Comment about the query) + pub fn query_functions( + &self, + target: Option<NetworkTargetId>, + source: Option<SourceId>, + guids: &[FunctionGUID], + ) -> Option<WarpFile<'static>> { + let query_functions_url = format!("{}/api/v1/functions/query", self.server_url); + let payload = Self::query_functions_body(target, source, guids); + + // Make the POST request + let response = self + .client + .post(&query_functions_url) + .json(&payload) + .send() + .ok()?; + if !response.status().is_success() { + log::error!("Failed to query functions: {}", response.status()); + return None; + } + + // Get response bytes and convert to WarpFile + let bytes = response.bytes().ok()?; + WarpFile::from_owned_bytes(bytes.to_vec()) + } + + /// Query the functions, returning the sources and the corresponding function guids. + /// + /// NOTE: **THIS IS BLOCKING** + /// + /// Route: `api/v1/functions/query/source` (TODO: Comment about the query) + pub fn query_functions_source( + &self, + target: Option<NetworkTargetId>, + guids: &[FunctionGUID], + ) -> Option<HashMap<SourceId, Vec<FunctionGUID>>> { + let query_functions_source_url = + format!("{}/api/v1/functions/query/source", self.server_url); + let payload = Self::query_functions_body(target, None, guids); + + // Make the POST request + let response = self + .client + .post(&query_functions_source_url) + .json(&payload) + .send() + .ok()?; + if !response.status().is_success() { + log::error!("Failed to query functions source: {}", response.status()); + return None; + } + + // Mapping of source id to function guids + let json_response: HashMap<String, Vec<String>> = response.json().ok()?; + let mapped_function_guids = json_response + .into_iter() + .filter_map(|(source_str, guid_strs)| { + let source_id = SourceId::from_str(&source_str).ok()?; + let guids = guid_strs + .into_iter() + .filter_map(|guid_str| FunctionGUID::from_str(&guid_str).ok()) + .collect(); + Some((source_id, guids)) + }) + .collect(); + + Some(mapped_function_guids) + } + + /// Pushes the file to the remote source. + /// + /// NOTE: **THIS IS BLOCKING** + /// + /// Route: `api/v1/files/{source}` + pub fn push_file(&self, source_id: SourceId, file: &WarpFile) -> bool { + let push_file_url = format!("{}/api/v1/files/{}", self.server_url, source_id.to_string()); + + // Convert WarpFile to bytes + let file_bytes = file.to_bytes(); + + // Create the form part with the file + let form = reqwest::blocking::multipart::Form::new().part( + "file", + reqwest::blocking::multipart::Part::bytes(file_bytes) + .file_name("data.warp") + .mime_str("application/octet-stream") + .unwrap(), + ); + + // Send the request + match self.client.post(&push_file_url).multipart(form).send() { + Ok(response) => { + if response.status().is_success() { + true + } else { + log::error!("Failed to push file: {}", response.status()); + false + } + } + Err(e) => { + log::error!("Failed to send push request: {}", e); + false + } + } + } +} diff --git a/plugins/warp/src/plugin.rs b/plugins/warp/src/plugin.rs index 3380c619..7f0481b8 100644 --- a/plugins/warp/src/plugin.rs +++ b/plugins/warp/src/plugin.rs @@ -3,6 +3,7 @@ use std::time::Instant; use crate::cache::container::add_cached_container; use crate::container::disk::DiskContainer; +use crate::container::network::{NetworkClient, NetworkContainer}; use crate::matcher::MatcherSettings; use crate::plugin::render_layer::HighlightRenderLayer; use crate::plugin::settings::PluginSettings; @@ -11,9 +12,11 @@ use binaryninja::background_task::BackgroundTask; use binaryninja::command::{ register_command, register_command_for_function, register_command_for_project, }; +use binaryninja::is_ui_enabled; use binaryninja::logger::Logger; use binaryninja::settings::Settings; use log::LevelFilter; +use reqwest::StatusCode; mod create; mod debug; @@ -26,6 +29,63 @@ mod render_layer; mod settings; mod workflow; +fn load_bundled_signatures() { + let global_bn_settings = Settings::new(); + let plugin_settings = PluginSettings::from_settings(&global_bn_settings); + // We want to load all the bundled directories into the container cache. + let background_task = BackgroundTask::new("Loading WARP files...", false); + let start = Instant::now(); + if plugin_settings.load_bundled_files { + let core_disk_container = DiskContainer::new_from_dir(core_signature_dir()); + log::debug!("{:#?}", core_disk_container); + add_cached_container(core_disk_container); + } + if plugin_settings.load_user_files { + let user_disk_container = DiskContainer::new_from_dir(user_signature_dir()); + log::debug!("{:#?}", user_disk_container); + add_cached_container(user_disk_container); + } + log::info!("Loading bundled files took {:?}", start.elapsed()); + background_task.finish(); +} + +fn load_network_container() { + let global_bn_settings = Settings::new(); + let plugin_settings = PluginSettings::from_settings(&global_bn_settings); + let background_task = BackgroundTask::new("Initializing WARP server...", false); + let start = Instant::now(); + if plugin_settings.enable_server { + let server_url = plugin_settings.server_url.clone(); + let server_api_key = plugin_settings.server_api_key.clone(); + let https_proxy_str = global_bn_settings.get_string("network.httpsProxy"); + let https_proxy = if https_proxy_str.is_empty() { + None + } else { + Some(https_proxy_str) + }; + match NetworkClient::new(server_url.clone(), server_api_key, https_proxy) { + Ok(network_client) => { + // Before constructing the container, let's make sure that the server is OK. + if let Ok(StatusCode::OK) = network_client.status() { + let network_container = NetworkContainer::new(network_client); + log::debug!("{:#?}", network_container); + add_cached_container(network_container); + } else { + log::error!( + "Server '{}' is not reachable, disabling container...", + server_url + ); + } + } + Err(e) => { + log::error!("Failed to add networked container: {}", e); + } + } + } + log::debug!("Initializing warp server took {:?}", start.elapsed()); + background_task.finish(); +} + #[no_mangle] #[allow(non_snake_case)] pub extern "C" fn CorePluginInit() -> bool { @@ -44,22 +104,21 @@ pub extern "C" fn CorePluginInit() -> bool { workflow::insert_workflow(); - let plugin_settings = PluginSettings::from_settings(&global_bn_settings); - // We want to load all the bundled directories into the container cache. - let background_task = BackgroundTask::new("Loading WARP files...", false); - let start = Instant::now(); - if plugin_settings.load_bundled_files { - let core_disk_container = DiskContainer::new_from_dir(core_signature_dir()); - log::debug!("{:#?}", core_disk_container); - add_cached_container(core_disk_container); + // TODO: Make the retrieval of containers wait on this to be done. + // TODO: We could also have a mechanism for lazily loading the files using the chunk header target. + // Loading bundled signatures might take a few hundred milliseconds. + if is_ui_enabled() { + std::thread::spawn(|| { + load_bundled_signatures(); + load_network_container(); + }); + } else { + load_bundled_signatures(); + std::thread::spawn(|| { + // Dependence on this is likely to not matter in headless, so we throw it on another thread. + load_network_container(); + }); } - if plugin_settings.load_user_files { - let user_disk_container = DiskContainer::new_from_dir(user_signature_dir()); - log::debug!("{:#?}", user_disk_container); - add_cached_container(user_disk_container); - } - log::info!("Loading bundled files took {:?}", start.elapsed()); - background_task.finish(); register_command( "WARP\\Run Matcher", diff --git a/plugins/warp/src/plugin/ffi/container.rs b/plugins/warp/src/plugin/ffi/container.rs index 21ad4dc0..4de6bab3 100644 --- a/plugins/warp/src/plugin/ffi/container.rs +++ b/plugins/warp/src/plugin/ffi/container.rs @@ -36,6 +36,27 @@ pub unsafe extern "C" fn BNWARPContainerGetName(container: *mut BNWARPContainer) } #[no_mangle] +pub unsafe extern "C" fn BNWARPContainerFetchFunctions( + container: *mut BNWARPContainer, + target: *mut BNWARPTarget, + guids: *const BNWARPFunctionGUID, + count: usize, +) { + let arc_container = ManuallyDrop::new(Arc::from_raw(container)); + let Ok(mut container) = arc_container.write() else { + return; + }; + + let target = unsafe { ManuallyDrop::new(Arc::from_raw(target)) }; + + let guids = unsafe { std::slice::from_raw_parts(guids, count) }; + + if let Err(e) = container.fetch_functions(&target, guids) { + log::error!("Failed to fetch functions: {}", e); + } +} + +#[no_mangle] pub unsafe extern "C" fn BNWARPContainerGetSources( container: *mut BNWARPContainer, count: *mut usize, diff --git a/plugins/warp/src/plugin/settings.rs b/plugins/warp/src/plugin/settings.rs index 6cd2e0cc..489deb23 100644 --- a/plugins/warp/src/plugin/settings.rs +++ b/plugins/warp/src/plugin/settings.rs @@ -19,7 +19,7 @@ pub struct PluginSettings { /// The API key to use for the selected WARP server, if not specified, you will be unable to push data and may be rate-limited. /// /// This is set to [PluginSettings::SERVER_API_KEY_DEFAULT] by default. - pub server_api_key: String, + pub server_api_key: Option<String>, /// Whether to allow networked WARP requests. Turning this off will not disable local WARP functionality. /// /// This is set to [PluginSettings::ENABLE_SERVER_DEFAULT] by default. @@ -33,9 +33,9 @@ impl PluginSettings { pub const LOAD_USER_FILES_SETTING: &'static str = "analysis.warp.loadUserFiles"; pub const SERVER_URL_DEFAULT: &'static str = "https://warp.binary.ninja"; pub const SERVER_URL_SETTING: &'static str = "analysis.warp.serverUrl"; - pub const SERVER_API_KEY_DEFAULT: &'static str = ""; + pub const SERVER_API_KEY_DEFAULT: Option<String> = None; pub const SERVER_API_KEY_SETTING: &'static str = "analysis.warp.serverApiKey"; - pub const ENABLE_SERVER_DEFAULT: bool = true; + pub const ENABLE_SERVER_DEFAULT: bool = false; pub const ENABLE_SERVER_SETTING: &'static str = "network.enableWARP"; pub fn register(bn_settings: &mut BNSettings) { @@ -44,7 +44,8 @@ impl PluginSettings { "type" : "boolean", "default" : Self::LOAD_BUNDLED_FILES_DEFAULT, "description" : "Whether to load bundled WARP files on startup. Turn this off if you want to manually load them.", - "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] + "ignore" : ["SettingsProjectScope", "SettingsResourceScope"], + "requiresRestart" : true }); bn_settings.register_setting_json( Self::LOAD_BUNDLED_FILES_SETTING, @@ -55,7 +56,8 @@ impl PluginSettings { "type" : "boolean", "default" : Self::LOAD_USER_FILES_DEFAULT, "description" : "Whether to load user WARP files on startup. Turn this off if you want to manually load them.", - "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] + "ignore" : ["SettingsProjectScope", "SettingsResourceScope"], + "requiresRestart" : true }); bn_settings.register_setting_json( Self::LOAD_USER_FILES_SETTING, @@ -66,7 +68,8 @@ impl PluginSettings { "type" : "string", "default" : Self::SERVER_URL_DEFAULT, "description" : "The WARP server to use.", - "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] + "ignore" : ["SettingsProjectScope", "SettingsResourceScope"], + "requiresRestart" : true }); bn_settings.register_setting_json(Self::SERVER_URL_SETTING, &server_url_prop.to_string()); let server_api_key_prop = json!({ @@ -75,7 +78,8 @@ impl PluginSettings { "default" : Self::SERVER_API_KEY_DEFAULT, "description" : "The API key to use for the selected WARP server, if not specified you will be unable to push data, and may be rate limited.", "ignore" : ["SettingsProjectScope", "SettingsResourceScope"], - "hidden": true + "hidden": true, + "requiresRestart" : true }); bn_settings.register_setting_json( Self::SERVER_API_KEY_SETTING, @@ -86,7 +90,8 @@ impl PluginSettings { "type" : "boolean", "default" : Self::ENABLE_SERVER_DEFAULT, "description" : "Whether or not to allow networked WARP requests. Turning this off will not disable local WARP functionality.", - "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] + "ignore" : ["SettingsProjectScope", "SettingsResourceScope"], + "requiresRestart" : true }); bn_settings.register_setting_json( Self::ENABLE_SERVER_SETTING, @@ -107,7 +112,10 @@ impl PluginSettings { settings.server_url = bn_settings.get_string(Self::SERVER_URL_SETTING); } if bn_settings.contains(Self::SERVER_API_KEY_SETTING) { - settings.server_url = bn_settings.get_string(Self::SERVER_API_KEY_SETTING); + let server_api_key_str = bn_settings.get_string(Self::SERVER_API_KEY_SETTING); + if !server_api_key_str.is_empty() { + settings.server_api_key = Some(server_api_key_str); + } } if bn_settings.contains(Self::ENABLE_SERVER_SETTING) { settings.enable_server = bn_settings.get_bool(Self::ENABLE_SERVER_SETTING); @@ -122,7 +130,7 @@ impl Default for PluginSettings { load_bundled_files: PluginSettings::LOAD_BUNDLED_FILES_DEFAULT, load_user_files: PluginSettings::LOAD_USER_FILES_DEFAULT, server_url: PluginSettings::SERVER_URL_DEFAULT.to_string(), - server_api_key: PluginSettings::SERVER_API_KEY_DEFAULT.to_string(), + server_api_key: PluginSettings::SERVER_API_KEY_DEFAULT, enable_server: PluginSettings::ENABLE_SERVER_DEFAULT, } } diff --git a/plugins/warp/src/plugin/workflow.rs b/plugins/warp/src/plugin/workflow.rs index 2d76222c..c2f82bd9 100644 --- a/plugins/warp/src/plugin/workflow.rs +++ b/plugins/warp/src/plugin/workflow.rs @@ -60,6 +60,15 @@ impl Command for RunMatcher { fn action(&self, view: &BinaryView) { let view = view.to_owned(); std::thread::spawn(move || { + // For embedded targets the user may not have set the sections up. + // Alert the user if we have no actual regions (+1 comes from the synthetic section). + let regions = relocatable_regions(&view); + if regions.len() <= 1 && view.memory_map().is_activated() { + log::warn!( + "No relocatable regions found, for best results please define sections for the binary!" + ); + } + run_matcher(&view); }); } @@ -75,14 +84,6 @@ pub fn run_matcher(view: &BinaryView) { let _ = get_warp_tag_type(view); view.file().forget_undo_actions(&undo_id); - // Alert the user if we have no actual regions (one comes from the synthetic section). - let regions = relocatable_regions(view); - if regions.len() <= 1 && view.memory_map().is_activated() { - log::warn!( - "No relocatable regions found, for best results please define sections for the binary!" - ); - } - // Then we want to actually find matching functions. let background_task = BackgroundTask::new("Matching on WARP functions...", true); let start = Instant::now(); diff --git a/plugins/warp/ui/matches.cpp b/plugins/warp/ui/matches.cpp index fad729d6..ed77085a 100644 --- a/plugins/warp/ui/matches.cpp +++ b/plugins/warp/ui/matches.cpp @@ -5,6 +5,7 @@ #include <QClipboard> #include <QFormLayout> +#include <thread> #include "theme.h" #include "warp.h" @@ -15,6 +16,8 @@ WarpCurrentFunctionWidget::WarpCurrentFunctionWidget(FunctionRef current) // NOTE: Might be nullptr if the no selected function. m_current = current; + m_logger = new BinaryNinja::Logger("WARP"); + // Create the QT stuff QGridLayout *layout = new QGridLayout(this); layout->setContentsMargins(2, 2, 2, 2); @@ -110,6 +113,26 @@ void WarpCurrentFunctionWidget::SetCurrentFunction(FunctionRef current) return; m_current = current; m_infoWidget->SetAnalysisFunction(m_current); + + if (current) + { + // Add the function to the processing list only if we have no already done so. + // If a user goes to a function, then navigates away, they do not want to + // have us try and send a network request! + { + std::lock_guard<std::mutex> lock(m_requestMutex); + uint64_t funcStart = current->GetStart(); + if (m_processedFunctions.find(funcStart) == m_processedFunctions.end()) { + m_pendingRequests.push_back(current); + } + } + if (!m_requestInProgress.exchange(true)) { + BinaryNinja::WorkerPriorityEnqueue([this]() { + ProcessPendingFetchRequests(); + }); + } + } + UpdateMatches(); } @@ -162,3 +185,51 @@ void WarpCurrentFunctionWidget::UpdateMatches() m_tableWidget->SetFunctions(matches); } + +void WarpCurrentFunctionWidget::ProcessPendingFetchRequests() +{ + std::vector<FunctionRef> requests; + { + std::lock_guard<std::mutex> lock(m_requestMutex); + requests = std::move(m_pendingRequests); + m_pendingRequests.clear(); + } + + if (requests.empty()) { + m_requestInProgress = false; + return; + } + + auto start_time = std::chrono::high_resolution_clock::now(); + + std::vector<Warp::FunctionGUID> guids; + Warp::Ref<Warp::Target> target; + for (const auto& func : requests) { + // TODO: Need to send multiple requests if there is multiple targets. + if (!target) + target = Warp::Target::FromPlatform(*func->GetPlatform()); + if (const auto guid = Warp::GetAnalysisFunctionGUID(*func); guid.has_value()) + guids.push_back(guid.value()); + } + + // Actually fetch the data! + if (!guids.empty()) + for (const auto &container: Warp::Container::All()) + container->FetchFunctions(*target, guids); + + { + std::lock_guard<std::mutex> lock(m_requestMutex); + for (const auto& func : requests) { + m_processedFunctions.insert(func->GetStart()); + } + } + + // TODO: Update the matches, make sure there was stuff added first lol. + // TODO: UpdateMatches(); + + const auto end_time = std::chrono::high_resolution_clock::now(); + const std::chrono::duration<double> elapsed_time = end_time - start_time; + m_logger->LogDebug("ProcessPendingRequests took %f seconds", elapsed_time.count()); + + m_requestInProgress = false; +} diff --git a/plugins/warp/ui/matches.h b/plugins/warp/ui/matches.h index 4021f10c..251dcb89 100644 --- a/plugins/warp/ui/matches.h +++ b/plugins/warp/ui/matches.h @@ -16,6 +16,13 @@ class WarpCurrentFunctionWidget : public QWidget WarpFunctionTableWidget *m_tableWidget; WarpFunctionInfoWidget *m_infoWidget; + LoggerRef m_logger; + + std::mutex m_requestMutex; + std::vector<FunctionRef> m_pendingRequests; + std::atomic<bool> m_requestInProgress {false}; + std::unordered_set<uint64_t> m_processedFunctions; + public: explicit WarpCurrentFunctionWidget(FunctionRef current); @@ -26,4 +33,6 @@ public: FunctionRef GetCurrentFunction() { return m_current; }; void UpdateMatches(); + + void ProcessPendingFetchRequests(); }; diff --git a/plugins/warp/ui/plugin.cpp b/plugins/warp/ui/plugin.cpp index 69aaaf34..923a79b3 100644 --- a/plugins/warp/ui/plugin.cpp +++ b/plugins/warp/ui/plugin.cpp @@ -170,6 +170,9 @@ void WarpSidebarWidget::notifyViewChanged(ViewFrame *view) void WarpSidebarWidget::notifyViewLocationChanged(View *view, const ViewLocation &location) { + // Warp sidebar really should only update if it is visible, otherwise its a waste of cycles. + if (!this->isVisible()) + return; auto function = location.getFunction(); // TODO: Only update if the function exists? // NOTE: The function called will exit early if it is the same function. |