WARP 1.0

- Added FFI
- Added a sidebar to the UI
- Added project, directory and archive processing
- Added generic `Container` interface for extensible stores of WARP data
- Fixed type references being constructed and pulled incorrectly
- Added HTML, Markdown and JSON report generation
- Made the WARP information added as an analysis activity
- Flattened the signatures directory, the target information is stored in the file now
- Matched function information is stored as function metadata in the database to reliably persist, alongside the function GUID
- Split the matching out from the application, allowing you to match on a given function without applying it
- Added more/better tests
- Added support for binaries with multiple architectures, the functions are now also queried based off the Target, see WARP spec for more details
- Greatly improved support for RISC architectures, see WARP spec for more details
- Greatly improved UX when loading files after the fact, will now sanely rerun the matcher
- Omitted the function type if not a user type, this greatly reduces file size
- Improved support for functions that reference a page aligned base pointer, see WARP spec for more details
- Removed some extra cache structures that were causing erroneous behavior
- Fixed edge-case in LLIL traversal missing some constant pointers, this was a bug in the Rust bindings
- Added support for function comments
- Made long running tasks, such as generating, matching and loading signatures, cancellable where possible
- Made function constraints more versatile, allowing for easy extensions in the future, see WARP spec for details
- Added options to signature generation, such as what data to store, and whether to compress the data or not
- Made all long running tasks prompt the user for required information before the task starts, allowing users to "set it and forget it" and not have to baby sit the finalization of the task
- Myriad of other changes to the actual WARP format that impact performance, file size and general feature set, see https://github.com/Vector35/warp for more details

1 files changed, 307 insertions, 0 deletions

diff --git a/plugins/warp/src/container/memory.rs b/plugins/warp/src/container/memory.rs
new file mode 100644
index 00000000..cf53390e
--- /dev/null
+++ b/plugins/warp/src/container/memory.rs
@@ -0,0 +1,307 @@
+use crate::container::{Container, ContainerError, ContainerResult, SourceId, SourcePath};
+use std::collections::HashMap;
+use std::fmt::Display;
+use warp::r#type::guid::TypeGUID;
+use warp::r#type::{ComputedType, Type};
+use warp::signature::function::{Function, FunctionGUID};
+use warp::target::Target;
+
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
+pub struct MemoryContainer {
+    sources: HashMap<SourceId, MemorySource>,
+}
+
+impl MemoryContainer {
+    pub fn new() -> Self {
+        MemoryContainer::default()
+    }
+
+    pub fn with_source(mut self, id: SourceId, source: MemorySource) -> Self {
+        self.sources.insert(id, source);
+        self
+    }
+
+    pub fn with_source_function(
+        mut self,
+        id: SourceId,
+        guid: FunctionGUID,
+        func: Function,
+    ) -> Self {
+        self.sources
+            .entry(id)
+            .or_default()
+            .functions
+            .entry(guid)
+            .or_default()
+            .push(func);
+        self
+    }
+
+    pub fn with_source_type(mut self, id: SourceId, guid: TypeGUID, ty: Type) -> Self {
+        self.sources.entry(id).or_default().types.insert(guid, ty);
+        self
+    }
+}
+
+impl Container for MemoryContainer {
+    fn sources(&self) -> ContainerResult<Vec<SourceId>> {
+        todo!()
+    }
+
+    fn add_source(&mut self, path: SourcePath) -> ContainerResult<SourceId> {
+        Err(ContainerError::CannotCreateSource(path))
+    }
+
+    fn commit_source(&mut self, _source: &SourceId) -> ContainerResult<bool> {
+        Ok(false)
+    }
+
+    fn is_source_writable(&self, source: &SourceId) -> ContainerResult<bool> {
+        let memory_source = self
+            .sources
+            .get(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        Ok(memory_source.writable)
+    }
+
+    fn is_source_uncommitted(&self, source: &SourceId) -> ContainerResult<bool> {
+        let _memory_source = self
+            .sources
+            .get(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        // NOTE: Memory containers do not have a notion of uncommitted data.
+        Ok(false)
+    }
+
+    fn source_path(&self, source: &SourceId) -> ContainerResult<SourcePath> {
+        Err(ContainerError::SourcePathUnavailable(*source))
+    }
+
+    fn add_computed_types(
+        &mut self,
+        source: &SourceId,
+        types: &[ComputedType],
+    ) -> ContainerResult<()> {
+        let memory_source = self
+            .sources
+            .get_mut(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        match memory_source.writable {
+            true => {
+                for ty in types {
+                    memory_source.types.insert(ty.guid, ty.ty.clone());
+                }
+                Ok(())
+            }
+            false => Err(ContainerError::SourceNotWritable(*source)),
+        }
+    }
+
+    fn remove_types(&mut self, source: &SourceId, guids: &[TypeGUID]) -> ContainerResult<()> {
+        let memory_source = self
+            .sources
+            .get_mut(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        match memory_source.writable {
+            true => {
+                for guid in guids {
+                    memory_source.types.remove(guid);
+                }
+                Ok(())
+            }
+            false => Err(ContainerError::SourceNotWritable(*source)),
+        }
+    }
+
+    fn add_functions(
+        &mut self,
+        _target: &Target,
+        source: &SourceId,
+        functions: &[Function],
+    ) -> ContainerResult<()> {
+        let memory_source = self
+            .sources
+            .get_mut(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        match memory_source.writable {
+            true => {
+                for function in functions {
+                    memory_source
+                        .functions
+                        .entry(function.guid)
+                        .or_default()
+                        .push(function.clone());
+                }
+                Ok(())
+            }
+            false => Err(ContainerError::SourceNotWritable(*source)),
+        }
+    }
+
+    fn remove_functions(
+        &mut self,
+        _target: &Target,
+        source: &SourceId,
+        functions: &[Function],
+    ) -> ContainerResult<()> {
+        let memory_source = self
+            .sources
+            .get_mut(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        match memory_source.writable {
+            true => {
+                for function in functions {
+                    if let Some(src_funcs) = memory_source.functions.get_mut(&function.guid) {
+                        src_funcs.retain(|f| f != function);
+                        if src_funcs.is_empty() {
+                            memory_source.functions.remove(&function.guid);
+                        }
+                    }
+                }
+                Ok(())
+            }
+            false => Err(ContainerError::SourceNotWritable(*source)),
+        }
+    }
+
+    fn sources_with_type_guid(&self, guid: &TypeGUID) -> ContainerResult<Vec<SourceId>> {
+        let sources = self
+            .sources
+            .iter()
+            .filter(|(_, source)| source.has_type_with_guid(guid))
+            .map(|(id, _)| *id)
+            .collect();
+        Ok(sources)
+    }
+
+    fn sources_with_type_guids(
+        &self,
+        guids: &[TypeGUID],
+    ) -> ContainerResult<HashMap<TypeGUID, Vec<SourceId>>> {
+        let mut result: HashMap<TypeGUID, Vec<SourceId>> = HashMap::new();
+        for (source_id, source) in &self.sources {
+            guids
+                .iter()
+                .filter(|guid| source.has_type_with_guid(guid))
+                .for_each(|guid| result.entry(*guid).or_default().push(*source_id));
+        }
+        Ok(result)
+    }
+
+    fn type_guids_with_name(
+        &self,
+        source: &SourceId,
+        name: &str,
+    ) -> ContainerResult<Vec<TypeGUID>> {
+        let memory_source = self
+            .sources
+            .get(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        Ok(memory_source.type_guids_with_name(name))
+    }
+
+    fn type_with_guid(&self, source: &SourceId, guid: &TypeGUID) -> ContainerResult<Option<Type>> {
+        let memory_source = self
+            .sources
+            .get(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        Ok(memory_source.type_with_guid(guid))
+    }
+
+    fn sources_with_function_guid(
+        &self,
+        _target: &Target,
+        guid: &FunctionGUID,
+    ) -> ContainerResult<Vec<SourceId>> {
+        let sources = self
+            .sources
+            .iter()
+            .filter(|(_, source)| source.has_function_with_guid(guid))
+            .map(|(id, _)| *id)
+            .collect();
+        Ok(sources)
+    }
+
+    fn sources_with_function_guids(
+        &self,
+        _target: &Target,
+        guids: &[FunctionGUID],
+    ) -> ContainerResult<HashMap<FunctionGUID, Vec<SourceId>>> {
+        let mut result: HashMap<FunctionGUID, Vec<SourceId>> = HashMap::new();
+        for (source_id, source) in &self.sources {
+            guids
+                .iter()
+                .filter(|guid| source.has_function_with_guid(guid))
+                .for_each(|guid| result.entry(*guid).or_default().push(*source_id));
+        }
+        Ok(result)
+    }
+
+    fn functions_with_guid(
+        &self,
+        _target: &Target,
+        source: &SourceId,
+        guid: &FunctionGUID,
+    ) -> ContainerResult<Vec<Function>> {
+        let memory_source = self
+            .sources
+            .get(source)
+            .ok_or(ContainerError::SourceNotFound(*source))?;
+        Ok(memory_source.functions_with_guid(guid))
+    }
+}
+
+impl Display for MemoryContainer {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("MemoryContainer")
+    }
+}
+
+/// An in-memory store of functions.
+///
+/// This is typically an overlay on top of a container source.
+#[derive(Eq, PartialEq, Debug, Clone)]
+pub struct MemorySource {
+    pub writable: bool,
+    pub functions: HashMap<FunctionGUID, Vec<Function>>,
+    pub types: HashMap<TypeGUID, Type>,
+    pub named_types: HashMap<String, Vec<TypeGUID>>,
+}
+
+impl MemorySource {
+    pub fn type_guids_with_name(&self, name: &str) -> Vec<TypeGUID> {
+        // TODO: The function here is a little goofy.
+        // TODO: This is cloned.
+        self.named_types.get(name).cloned().unwrap_or_default()
+    }
+
+    pub fn type_with_guid(&self, guid: &TypeGUID) -> Option<Type> {
+        // TODO: This is cloned.
+        self.types.get(guid).cloned()
+    }
+
+    pub fn functions_with_guid(&self, guid: &FunctionGUID) -> Vec<Function> {
+        // TODO: The function here is a little goofy.
+        // TODO: This is cloned.
+        self.functions.get(guid).cloned().unwrap_or_default()
+    }
+
+    pub fn has_type_with_guid(&self, guid: &TypeGUID) -> bool {
+        self.type_with_guid(guid).is_some()
+    }
+
+    pub fn has_function_with_guid(&self, guid: &FunctionGUID) -> bool {
+        !self.functions_with_guid(guid).is_empty()
+    }
+}
+
+impl Default for MemorySource {
+    fn default() -> Self {
+        Self {
+            writable: true,
+            functions: HashMap::new(),
+            types: HashMap::new(),
+            named_types: HashMap::new(),
+        }
+    }
+}