WARP 1.0

- Added FFI
- Added a sidebar to the UI
- Added project, directory and archive processing
- Added generic `Container` interface for extensible stores of WARP data
- Fixed type references being constructed and pulled incorrectly
- Added HTML, Markdown and JSON report generation
- Made the WARP information added as an analysis activity
- Flattened the signatures directory, the target information is stored in the file now
- Matched function information is stored as function metadata in the database to reliably persist, alongside the function GUID
- Split the matching out from the application, allowing you to match on a given function without applying it
- Added more/better tests
- Added support for binaries with multiple architectures, the functions are now also queried based off the Target, see WARP spec for more details
- Greatly improved support for RISC architectures, see WARP spec for more details
- Greatly improved UX when loading files after the fact, will now sanely rerun the matcher
- Omitted the function type if not a user type, this greatly reduces file size
- Improved support for functions that reference a page aligned base pointer, see WARP spec for more details
- Removed some extra cache structures that were causing erroneous behavior
- Fixed edge-case in LLIL traversal missing some constant pointers, this was a bug in the Rust bindings
- Added support for function comments
- Made long running tasks, such as generating, matching and loading signatures, cancellable where possible
- Made function constraints more versatile, allowing for easy extensions in the future, see WARP spec for details
- Added options to signature generation, such as what data to store, and whether to compress the data or not
- Made all long running tasks prompt the user for required information before the task starts, allowing users to "set it and forget it" and not have to baby sit the finalization of the task
- Myriad of other changes to the actual WARP format that impact performance, file size and general feature set, see https://github.com/Vector35/warp for more details

1 files changed, 378 insertions, 0 deletions

diff --git a/plugins/warp/api/warp.h b/plugins/warp/api/warp.h
new file mode 100644
index 00000000..b57a0c7b
--- /dev/null
+++ b/plugins/warp/api/warp.h
@@ -0,0 +1,378 @@
+#pragma once
+
+#include <binaryninjaapi.h>
+#include "warpcore.h"
+
+template<class T, T *(*AddObjectReference)(T *), void (*FreeObjectReference)(T *)>
+class WarpRefCountObject
+{
+	void AddRefInternal() { m_refs.fetch_add(1); }
+
+	void ReleaseInternal()
+	{
+		if (m_refs.fetch_sub(1) == 1)
+		{
+			if (!m_registeredRef)
+				delete this;
+		}
+	}
+
+public:
+	std::atomic<int> m_refs;
+	bool m_registeredRef = false;
+	T *m_object;
+
+	WarpRefCountObject() : m_refs(0), m_object(nullptr)
+	{
+	}
+
+	virtual ~WarpRefCountObject() = default;
+
+	T *GetObject() const { return m_object; }
+
+	static T *GetObject(WarpRefCountObject *obj)
+	{
+		if (!obj)
+			return nullptr;
+		return obj->GetObject();
+	}
+
+	void AddRef()
+	{
+		if (m_object && (m_refs != 0))
+			AddObjectReference(m_object);
+		AddRefInternal();
+	}
+
+	void Release()
+	{
+		if (m_object)
+			FreeObjectReference(m_object);
+		ReleaseInternal();
+	}
+
+	void AddRefForRegistration() { m_registeredRef = true; }
+
+	void ReleaseForRegistration()
+	{
+		m_object = nullptr;
+		m_registeredRef = false;
+		if (m_refs == 0)
+			delete this;
+	}
+};
+
+namespace Warp {
+	template<class T>
+	class Ref
+	{
+		T *m_obj;
+#ifdef BN_REF_COUNT_DEBUG
+		void* m_assignmentTrace = nullptr;
+#endif
+
+	public:
+		Ref() : m_obj(NULL)
+		{
+		}
+
+		Ref(T *obj) : m_obj(obj)
+		{
+			if (m_obj)
+			{
+				m_obj->AddRef();
+#ifdef BN_REF_COUNT_DEBUG
+				m_assignmentTrace = BNRegisterObjectRefDebugTrace(typeid(T).name());
+#endif
+			}
+		}
+
+		Ref(const Ref &obj) : m_obj(obj.m_obj)
+		{
+			if (m_obj)
+			{
+				m_obj->AddRef();
+#ifdef BN_REF_COUNT_DEBUG
+				m_assignmentTrace = BNRegisterObjectRefDebugTrace(typeid(T).name());
+#endif
+			}
+		}
+
+		Ref(Ref &&other) : m_obj(other.m_obj)
+		{
+			other.m_obj = 0;
+#ifdef BN_REF_COUNT_DEBUG
+			m_assignmentTrace = other.m_assignmentTrace;
+#endif
+		}
+
+		~Ref()
+		{
+			if (m_obj)
+			{
+				m_obj->Release();
+#ifdef BN_REF_COUNT_DEBUG
+				BNUnregisterObjectRefDebugTrace(typeid(T).name(), m_assignmentTrace);
+#endif
+			}
+		}
+
+		Ref<T> &operator=(const Ref<T> &obj)
+		{
+#ifdef BN_REF_COUNT_DEBUG
+			if (m_obj)
+				BNUnregisterObjectRefDebugTrace(typeid(T).name(), m_assignmentTrace);
+			if (obj.m_obj)
+				m_assignmentTrace = BNRegisterObjectRefDebugTrace(typeid(T).name());
+#endif
+			T *oldObj = m_obj;
+			m_obj = obj.m_obj;
+			if (m_obj)
+				m_obj->AddRef();
+			if (oldObj)
+				oldObj->Release();
+			return *this;
+		}
+
+		Ref<T> &operator=(Ref<T> &&other)
+		{
+			if (m_obj)
+			{
+#ifdef BN_REF_COUNT_DEBUG
+				BNUnregisterObjectRefDebugTrace(typeid(T).name(), m_assignmentTrace);
+#endif
+				m_obj->Release();
+			}
+			m_obj = other.m_obj;
+			other.m_obj = 0;
+#ifdef BN_REF_COUNT_DEBUG
+			m_assignmentTrace = other.m_assignmentTrace;
+#endif
+			return *this;
+		}
+
+		Ref<T> &operator=(T *obj)
+		{
+#ifdef BN_REF_COUNT_DEBUG
+			if (m_obj)
+				BNUnregisterObjectRefDebugTrace(typeid(T).name(), m_assignmentTrace);
+			if (obj)
+				m_assignmentTrace = BNRegisterObjectRefDebugTrace(typeid(T).name());
+#endif
+			T *oldObj = m_obj;
+			m_obj = obj;
+			if (m_obj)
+				m_obj->AddRef();
+			if (oldObj)
+				oldObj->Release();
+			return *this;
+		}
+
+		operator T *() const
+		{
+			return m_obj;
+		}
+
+		T *operator->() const
+		{
+			return m_obj;
+		}
+
+		T &operator*() const
+		{
+			return *m_obj;
+		}
+
+		bool operator!() const
+		{
+			return m_obj == NULL;
+		}
+
+		bool operator==(const T *obj) const
+		{
+			return T::GetObject(m_obj) == T::GetObject(obj);
+		}
+
+		bool operator==(const Ref<T> &obj) const
+		{
+			return T::GetObject(m_obj) == T::GetObject(obj.m_obj);
+		}
+
+		bool operator!=(const T *obj) const
+		{
+			return T::GetObject(m_obj) != T::GetObject(obj);
+		}
+
+		bool operator!=(const Ref<T> &obj) const
+		{
+			return T::GetObject(m_obj) != T::GetObject(obj.m_obj);
+		}
+
+		bool operator<(const T *obj) const
+		{
+			return T::GetObject(m_obj) < T::GetObject(obj);
+		}
+
+		bool operator<(const Ref<T> &obj) const
+		{
+			return T::GetObject(m_obj) < T::GetObject(obj.m_obj);
+		}
+
+		T *GetPtr() const
+		{
+			return m_obj;
+		}
+	};
+
+	class WarpUUID
+	{
+		BNWARPUUID uuid;
+
+	public:
+		WarpUUID() = default;
+		
+		WarpUUID(BNWARPUUID uuid) : uuid(uuid) {}
+
+		std::string ToString() const;
+
+		bool operator==(const WarpUUID &other) const
+		{
+			return BNWARPUUIDEqual(&uuid, &other.uuid);
+		}
+
+		bool operator!=(const WarpUUID &other) const
+		{
+			return !(*this == other);
+		}
+
+		BNWARPUUID* RawMut()
+		{
+			return &uuid;
+		}
+
+		const BNWARPUUID* Raw() const
+		{
+			return &uuid;
+		}
+	};
+
+	typedef WarpUUID Source;
+	typedef WarpUUID BasicBlockGUID;
+	typedef WarpUUID FunctionGUID;
+	typedef WarpUUID ConstraintGUID;
+	typedef WarpUUID TypeGUID;
+
+	class Target : public WarpRefCountObject<BNWARPTarget, BNWARPNewTargetReference,
+				BNWARPFreeTargetReference>
+	{
+	public:
+		explicit Target(BNWARPTarget *target);
+
+		static Ref<Target> FromPlatform(const BinaryNinja::Platform& platform); 
+	};
+	
+	struct Constraint
+	{
+		ConstraintGUID guid;
+		std::optional<int64_t> offset;
+
+		Constraint(ConstraintGUID guid, std::optional<int64_t> offset);
+
+		static Constraint FromAPIObject(BNWARPConstraint* constraint);
+	};
+
+	struct FunctionComment
+	{
+		std::string text;
+		int64_t offset;
+
+		FunctionComment(std::string text, int64_t offset);
+
+		static FunctionComment FromAPIObject(BNWARPFunctionComment* comment);
+	};
+
+	class Function : public WarpRefCountObject<BNWARPFunction, BNWARPNewFunctionReference, BNWARPFreeFunctionReference>
+	{
+	public:
+		explicit Function(BNWARPFunction *function);
+
+		bool operator==(const Function &other) const
+		{
+			return BNWARPFunctionsEqual(m_object, other.m_object);
+		}
+
+		FunctionGUID GetGUID() const;
+
+		std::string GetSymbolName() const;
+
+		BinaryNinja::Ref<BinaryNinja::Symbol> GetSymbol(const BinaryNinja::Function &function) const;
+
+		BinaryNinja::Ref<BinaryNinja::Type> GetType(const BinaryNinja::Function &function) const;
+
+		std::vector<Constraint> GetConstraints() const;
+
+		std::vector<FunctionComment> GetComments() const;
+
+		static Ref<Function> Get(const BinaryNinja::Function &function);
+
+		static Ref<Function> GetMatched(const BinaryNinja::Function &function);
+
+		void Apply(const BinaryNinja::Function &function) const;
+
+		static void RemoveMatch(const BinaryNinja::Function &function);
+	};
+
+	class Container : public WarpRefCountObject<BNWARPContainer, BNWARPNewContainerReference,
+				BNWARPFreeContainerReference>
+	{
+	public:
+		explicit Container(BNWARPContainer *container);
+
+		/// Retrieve all available containers.
+		static std::vector<Ref<Container> > All();
+
+		std::string GetName() const;
+
+		std::vector<Source> GetSources() const;
+
+		std::optional<Source> AddSource(const std::string &sourcePath) const;
+
+		bool CommitSource(const Source &source) const;
+
+		bool IsSourceUncommitted(const Source &source) const;
+
+		bool IsSourceWritable(const Source &source) const;
+
+		std::optional<std::string> SourcePath(const Source &source) const;
+
+		bool AddFunctions(const Target &target, const Source &source, const std::vector<Ref<Function> > &functions) const;
+
+		bool AddTypes(const BinaryNinja::BinaryView &view, const Source &source,
+		              const std::vector<BinaryNinja::Ref<BinaryNinja::Type> > &types) const;
+
+		bool RemoveFunctions(const Target &target, const Source &source, const std::vector<Ref<Function> > &functions) const;
+
+		bool RemoveTypes(const Source &source, const std::vector<TypeGUID> &guids) const;
+
+		std::vector<Source> GetSourcesWithFunctionGUID(const Target& target, const FunctionGUID &guid) const;
+
+		std::vector<Source> GetSourcesWithTypeGUID(const TypeGUID &guid) const;
+
+		std::vector<Ref<Function> > GetFunctionsWithGUID(const Target& target, const Source &source, const FunctionGUID &guid) const;
+
+		BinaryNinja::Ref<BinaryNinja::Type> GetTypeWithGUID(const BinaryNinja::Architecture &arch, const Source &source,
+		                                                    const TypeGUID &guid) const;
+
+		std::vector<TypeGUID> GetTypeGUIDsWithName(const Source &source, const std::string &name) const;
+	};
+
+	void RunMatcher(const BinaryNinja::BinaryView& view);
+
+	bool IsInstructionVariant(const BinaryNinja::LowLevelILFunction &function, BinaryNinja::ExprId idx);
+
+	bool IsInstructionBlacklisted(const BinaryNinja::LowLevelILFunction &function, BinaryNinja::ExprId idx);
+	
+	std::optional<FunctionGUID> GetAnalysisFunctionGUID(const BinaryNinja::Function &function);
+
+	std::optional<BasicBlockGUID> GetBasicBlockGUID(const BinaryNinja::BasicBlock &basicBlock);
+}