Merge branch 'analysis_cache' into dev

6 files changed, 187 insertions, 10 deletions

diff --git a/binaryninjaapi.h b/binaryninjaapi.h
index f79e9d75..7c417623 100644
--- a/binaryninjaapi.h
+++ b/binaryninjaapi.h
@@ -511,8 +511,14 @@ namespace BinaryNinja
 
 		bool IsBackedByDatabase() const;
 		bool CreateDatabase(const std::string& name, BinaryView* data);
+		bool CreateDatabase(const std::string& name, BinaryView* data,
+			const std::function<void(size_t progress, size_t total)>& progressCallback);
 		Ref<BinaryView> OpenExistingDatabase(const std::string& path);
+		Ref<BinaryView> OpenExistingDatabase(const std::string& path,
+			const std::function<void(size_t progress, size_t total)>& progressCallback);
 		bool SaveAutoSnapshot(BinaryView* data);
+		bool SaveAutoSnapshot(BinaryView* data,
+			const std::function<void(size_t progress, size_t total)>& progressCallback);
 
 		void BeginUndoActions();
 		void CommitUndoActions();
@@ -766,7 +772,10 @@ namespace BinaryNinja
 		bool IsAnalysisChanged() const;
 		bool IsBackedByDatabase() const;
 		bool CreateDatabase(const std::string& path);
+		bool CreateDatabase(const std::string& path,
+			const std::function<void(size_t progress, size_t total)>& progressCallback);
 		bool SaveAutoSnapshot();
+		bool SaveAutoSnapshot(const std::function<void(size_t progress, size_t total)>& progressCallback);
 
 		void BeginUndoActions();
 		void AddUndoAction(UndoAction* action);
@@ -910,6 +919,8 @@ namespace BinaryNinja
 		void UndefineUserType(const std::string& name);
 
 		bool FindNextData(uint64_t start, const DataBuffer& data, uint64_t& result, BNFindFlag flags = NoFindFlags);
+
+		void Reanalyze();
 	};
 
 	class BinaryData: public BinaryView
@@ -1602,6 +1613,7 @@ namespace BinaryNinja
 		bool WasAutomaticallyDiscovered() const;
 		bool CanReturn() const;
 		bool HasExplicitlyDefinedType() const;
+		bool NeedsUpdate() const;
 
 		std::vector<Ref<BasicBlock>> GetBasicBlocks() const;
 		void MarkRecentUse();
@@ -1626,6 +1638,7 @@ namespace BinaryNinja
 		std::vector<uint32_t> GetRegistersReadByInstruction(Architecture* arch, uint64_t addr);
 		std::vector<uint32_t> GetRegistersWrittenByInstruction(Architecture* arch, uint64_t addr);
 		std::vector<StackVariableReference> GetStackVariablesReferencedByInstruction(Architecture* arch, uint64_t addr);
+		std::vector<BNConstantReference> GetConstantsReferencedByInstruction(Architecture* arch, uint64_t addr);
 
 		Ref<LowLevelILFunction> GetLiftedIL() const;
 		size_t GetLiftedILForInstruction(Architecture* arch, uint64_t addr);
@@ -1661,6 +1674,8 @@ namespace BinaryNinja
 			size_t operand);
 		void SetIntegerConstantDisplayType(Architecture* arch, uint64_t instrAddr, uint64_t value, size_t operand,
 			BNIntegerDisplayType type);
+
+		void Reanalyze();
 	};
 
 	struct FunctionGraphEdge
diff --git a/binaryninjacore.h b/binaryninjacore.h
index 2ba6f7e6..3aa0a1e6 100644
--- a/binaryninjacore.h
+++ b/binaryninjacore.h
@@ -1013,6 +1013,12 @@ extern "C"
 		void (*addAction)(void* ctxt, BNMainThreadAction* action);
 	};
 
+	struct BNConstantReference
+	{
+		int64_t value;
+		size_t size;
+	};
+
 	BINARYNINJACOREAPI char* BNAllocString(const char* contents);
 	BINARYNINJACOREAPI void BNFreeString(char* str);
 
@@ -1105,8 +1111,14 @@ extern "C"
 
 	BINARYNINJACOREAPI bool BNIsBackedByDatabase(BNFileMetadata* file);
 	BINARYNINJACOREAPI bool BNCreateDatabase(BNBinaryView* data, const char* path);
+	BINARYNINJACOREAPI bool BNCreateDatabaseWithProgress(BNBinaryView* data, const char* path,
+		void* ctxt, void (*progress)(void* ctxt, size_t progress, size_t total));
 	BINARYNINJACOREAPI BNBinaryView* BNOpenExistingDatabase(BNFileMetadata* file, const char* path);
+	BINARYNINJACOREAPI BNBinaryView* BNOpenExistingDatabaseWithProgress(BNFileMetadata* file, const char* path,
+		void* ctxt, void (*progress)(void* ctxt, size_t progress, size_t total));
 	BINARYNINJACOREAPI bool BNSaveAutoSnapshot(BNBinaryView* data);
+	BINARYNINJACOREAPI bool BNSaveAutoSnapshotWithProgress(BNBinaryView* data, void* ctxt,
+		void (*progress)(void* ctxt, size_t progress, size_t total));
 
 	BINARYNINJACOREAPI char* BNGetFilename(BNFileMetadata* file);
 	BINARYNINJACOREAPI void BNSetFilename(BNFileMetadata* file, const char* name);
@@ -1369,6 +1381,7 @@ extern "C"
 	BINARYNINJACOREAPI void BNRemoveUserFunction(BNBinaryView* view, BNFunction* func);
 	BINARYNINJACOREAPI void BNUpdateAnalysis(BNBinaryView* view);
 	BINARYNINJACOREAPI void BNAbortAnalysis(BNBinaryView* view);
+	BINARYNINJACOREAPI bool BNIsFunctionUpdateNeeded(BNFunction* func);
 
 	BINARYNINJACOREAPI BNFunction* BNNewFunctionReference(BNFunction* func);
 	BINARYNINJACOREAPI void BNFreeFunction(BNFunction* func);
@@ -1433,6 +1446,9 @@ extern "C"
 	BINARYNINJACOREAPI BNStackVariableReference* BNGetStackVariablesReferencedByInstruction(BNFunction* func, BNArchitecture* arch,
 	                                                                                        uint64_t addr, size_t* count);
 	BINARYNINJACOREAPI void BNFreeStackVariableReferenceList(BNStackVariableReference* refs, size_t count);
+	BINARYNINJACOREAPI BNConstantReference* BNGetConstantsReferencedByInstruction(BNFunction* func,
+		BNArchitecture* arch, uint64_t addr, size_t* count);
+	BINARYNINJACOREAPI void BNFreeConstantReferenceList(BNConstantReference* refs);
 
 	BINARYNINJACOREAPI BNLowLevelILFunction* BNGetFunctionLiftedIL(BNFunction* func);
 	BINARYNINJACOREAPI size_t BNGetLiftedILForInstruction(BNFunction* func, BNArchitecture* arch, uint64_t addr);
@@ -1549,6 +1565,9 @@ extern "C"
 	BINARYNINJACOREAPI void BNUndefineAnalysisType(BNBinaryView* view, const char* name);
 	BINARYNINJACOREAPI void BNUndefineUserAnalysisType(BNBinaryView* view, const char* name);
 
+	BINARYNINJACOREAPI void BNReanalyzeAllFunctions(BNBinaryView* view);
+	BINARYNINJACOREAPI void BNReanalyzeFunction(BNFunction* func);
+
 	// Disassembly settings
 	BINARYNINJACOREAPI BNDisassemblySettings* BNCreateDisassemblySettings(void);
 	BINARYNINJACOREAPI BNDisassemblySettings* BNNewDisassemblySettingsReference(BNDisassemblySettings* settings);
diff --git a/binaryview.cpp b/binaryview.cpp
index cff3ed10..6631f3d6 100644
--- a/binaryview.cpp
+++ b/binaryview.cpp
@@ -518,12 +518,25 @@ bool BinaryView::CreateDatabase(const string& path)
 }
 
 
+bool BinaryView::CreateDatabase(const string& path,
+	const function<void(size_t progress, size_t total)>& progressCallback)
+{
+	return m_file->CreateDatabase(path, this, progressCallback);
+}
+
+
 bool BinaryView::SaveAutoSnapshot()
 {
 	return m_file->SaveAutoSnapshot(this);
 }
 
 
+bool BinaryView::SaveAutoSnapshot(const function<void(size_t progress, size_t total)>& progressCallback)
+{
+	return m_file->SaveAutoSnapshot(this, progressCallback);
+}
+
+
 void BinaryView::BeginUndoActions()
 {
 	m_file->BeginUndoActions();
@@ -1427,6 +1440,12 @@ bool BinaryView::FindNextData(uint64_t start, const DataBuffer& data, uint64_t&
 }
 
 
+void BinaryView::Reanalyze()
+{
+	BNReanalyzeAllFunctions(m_object);
+}
+
+
 BinaryData::BinaryData(FileMetadata* file): BinaryView(BNCreateBinaryDataView(file->GetObject()))
 {
 }
diff --git a/filemetadata.cpp b/filemetadata.cpp
index 51e152bf..b3764167 100644
--- a/filemetadata.cpp
+++ b/filemetadata.cpp
@@ -25,6 +25,19 @@ using namespace Json;
 using namespace std;
 
 
+struct DatabaseProgressCallbackContext
+{
+	std::function<void(size_t, size_t)> func;
+};
+
+
+static void DatabaseProgressCallback(void* ctxt, size_t progress, size_t total)
+{
+	DatabaseProgressCallbackContext* cb = (DatabaseProgressCallbackContext*)ctxt;
+	cb->func(progress, total);
+}
+
+
 char* NavigationHandler::GetCurrentViewCallback(void* ctxt)
 {
 	NavigationHandler* handler = (NavigationHandler*)ctxt;
@@ -246,6 +259,15 @@ bool FileMetadata::CreateDatabase(const string& name, BinaryView* data)
 }
 
 
+bool FileMetadata::CreateDatabase(const string& name, BinaryView* data,
+	const function<void(size_t progress, size_t total)>& progressCallback)
+{
+	DatabaseProgressCallbackContext cb;
+	cb.func = progressCallback;
+	return BNCreateDatabaseWithProgress(data->GetObject(), name.c_str(), &cb, DatabaseProgressCallback);
+}
+
+
 Ref<BinaryView> FileMetadata::OpenExistingDatabase(const string& path)
 {
 	BNBinaryView* data = BNOpenExistingDatabase(m_object, path.c_str());
@@ -255,12 +277,33 @@ Ref<BinaryView> FileMetadata::OpenExistingDatabase(const string& path)
 }
 
 
+Ref<BinaryView> FileMetadata::OpenExistingDatabase(const string& path,
+	const function<void(size_t progress, size_t total)>& progressCallback)
+{
+	DatabaseProgressCallbackContext cb;
+	cb.func = progressCallback;
+	BNBinaryView* data = BNOpenExistingDatabaseWithProgress(m_object, path.c_str(), &cb, DatabaseProgressCallback);
+	if (!data)
+		return nullptr;
+	return new BinaryView(data);
+}
+
+
 bool FileMetadata::SaveAutoSnapshot(BinaryView* data)
 {
 	return BNSaveAutoSnapshot(data->GetObject());
 }
 
 
+bool FileMetadata::SaveAutoSnapshot(BinaryView* data,
+	const function<void(size_t progress, size_t total)>& progressCallback)
+{
+	DatabaseProgressCallbackContext cb;
+	cb.func = progressCallback;
+	return BNSaveAutoSnapshotWithProgress(data->GetObject(), &cb, DatabaseProgressCallback);
+}
+
+
 void FileMetadata::BeginUndoActions()
 {
 	BNBeginUndoActions(m_object);
diff --git a/function.cpp b/function.cpp
index c5033276..cbd9ec40 100644
--- a/function.cpp
+++ b/function.cpp
@@ -72,6 +72,12 @@ bool Function::HasExplicitlyDefinedType() const
 }
 
 
+bool Function::NeedsUpdate() const
+{
+	return BNIsFunctionUpdateNeeded(m_object);
+}
+
+
 vector<Ref<BasicBlock>> Function::GetBasicBlocks() const
 {
 	size_t count;
@@ -288,6 +294,19 @@ vector<StackVariableReference> Function::GetStackVariablesReferencedByInstructio
 }
 
 
+vector<BNConstantReference> Function::GetConstantsReferencedByInstruction(Architecture* arch, uint64_t addr)
+{
+	size_t count;
+	BNConstantReference* refs = BNGetConstantsReferencedByInstruction(m_object, arch->GetObject(), addr, &count);
+
+	vector<BNConstantReference> result;
+	result.insert(result.end(), &refs[0], &refs[count]);
+
+	BNFreeConstantReferenceList(refs);
+	return result;
+}
+
+
 Ref<LowLevelILFunction> Function::GetLiftedIL() const
 {
 	return new LowLevelILFunction(BNGetFunctionLiftedIL(m_object));
@@ -553,3 +572,9 @@ void Function::SetIntegerConstantDisplayType(Architecture* arch, uint64_t instrA
 {
 	BNSetIntegerConstantDisplayType(m_object, arch->GetObject(), instrAddr, value, operand, type);
 }
+
+
+void Function::Reanalyze()
+{
+	BNReanalyzeFunction(m_object);
+}
diff --git a/python/__init__.py b/python/__init__.py
index 027c2a1f..82aaba32 100644
--- a/python/__init__.py
+++ b/python/__init__.py
@@ -372,18 +372,32 @@ class FileMetadata(object):
 	def navigate(self, view, offset):
 		return core.BNNavigate(self.handle, str(view), offset)
 
-	def create_database(self, filename):
-
-		return core.BNCreateDatabase(self.raw.handle, str(filename))
+	def create_database(self, filename, progress_func = None):
+		if progress_func is None:
+			return core.BNCreateDatabase(self.raw.handle, str(filename))
+		else:
+			return core.BNCreateDatabaseWithProgress(self.raw.handle, str(filename), None,
+				ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_ulonglong, ctypes.c_ulonglong)(
+					lambda ctxt, cur, total: progress_func(cur, total)))
 
-	def open_existing_database(self, filename):
-		view = core.BNOpenExistingDatabase(self.handle, str(filename))
+	def open_existing_database(self, filename, progress_func = None):
+		if progress_func is None:
+			view = core.BNOpenExistingDatabase(self.handle, str(filename))
+		else:
+			view = core.BNOpenExistingDatabaseWithProgress(self.handle, str(filename), None,
+				ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_ulonglong, ctypes.c_ulonglong)(
+				lambda ctxt, cur, total: progress_func(cur, total)))
 		if view is None:
 			return None
 		return BinaryView(self, handle = view)
 
-	def save_auto_snapshot(self):
-		return core.BNSaveAutoSnapshot(self.raw.handle)
+	def save_auto_snapshot(self, progress_func = None):
+		if progress_func is None:
+			return core.BNSaveAutoSnapshot(self.raw.handle)
+		else:
+			return core.BNSaveAutoSnapshotWithProgress(self.raw.handle, None,
+				ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_ulonglong, ctypes.c_ulonglong)(
+				lambda ctxt, cur, total: progress_func(cur, total)))
 
 	def get_view_of_type(self, name):
 		view = core.BNGetFileViewOfType(self.handle, str(name))
@@ -1683,26 +1697,28 @@ class BinaryView(object):
 		"""
 		return core.LittleEndian
 
-	def create_database(self, filename):
+	def create_database(self, filename, progress_func = None):
 		"""
 		``perform_get_database`` writes the current database (.bndb) file out to the specified file.
 
 		:param str filename: path and filename to write the bndb to, this string `should` have ".bndb" appended to it.
+		:param callable() progress_func: optional function to be called with the current progress and total count.
 		:return: true on success, false on failure
 		:rtype: bool
 		"""
 		return self.file.create_database(filename)
 
-	def save_auto_snapshot(self):
+	def save_auto_snapshot(self, progress_func = None):
 		"""
 		``save_auto_snapshot`` saves the current database to the already created file.
 
 		.. note:: :py:method:`create_database` should have been called prior to executing this method
 
+		:param callable() progress_func: optional function to be called with the current progress and total count.
 		:return: True if it successfully saved the snapshot, False otherwise
 		:rtype: bool
 		"""
-		return self.file.save_auto_snapshot()
+		return self.file.save_auto_snapshot(progress_func)
 
 	def get_view_of_type(self, name):
 		"""
@@ -3215,6 +3231,14 @@ class BinaryView(object):
 			return None
 		return result.value
 
+	def reanalyze(self):
+		"""
+		``reanalyze`` causes all functions to be reanalyzed. This function does not wait for the analysis to finish.
+
+		:rtype: None
+		"""
+		core.BNReanalyzeAllFunctions(self.handle)
+
 	def __setattr__(self, name, value):
 		try:
 			object.__setattr__(self,name,value)
@@ -4248,6 +4272,16 @@ class StackVariableReference:
 			return "<operand %d ref to %s%+#x>" % (self.source_operand, self.name, self.referenced_offset)
 		return "<operand %d ref to %s>" % (self.source_operand, self.name)
 
+class ConstantReference:
+	def __init__(self, val, size):
+		self.value = val
+		self.size = size
+
+	def __repr__(self):
+		if self.size == 0:
+			return "<constant %#x>" % self.value
+		return "<constant %#x size %d>" % (self.value, self.size)
+
 class IndirectBranchInfo:
 	def __init__(self, source_arch, source_addr, dest_arch, dest_addr, auto_defined):
 		self.source_arch = source_arch
@@ -4323,6 +4357,11 @@ class Function(object):
 		return core.BNHasExplicitlyDefinedType(self.handle)
 
 	@property
+	def needs_update(self):
+		"""Whether the function has analysis that needs to be updated (read-only)"""
+		return core.BNIsFunctionUpdateNeeded(self.handle)
+
+	@property
 	def basic_blocks(self):
 		"""List of basic blocks (read-only)"""
 		count = ctypes.c_ulonglong()
@@ -4531,6 +4570,15 @@ class Function(object):
 		core.BNFreeStackVariableReferenceList(refs, count.value)
 		return result
 
+	def get_constants_referenced_by(self, arch, addr):
+		count = ctypes.c_ulonglong()
+		refs = core.BNGetConstantsReferencedByInstruction(self.handle, arch.handle, addr, count)
+		result = []
+		for i in xrange(0, count.value):
+			result.append(ConstantReference(refs[i].value, refs[i].size))
+		core.BNFreeConstantReferenceList(refs)
+		return result
+
 	def get_lifted_il_at(self, arch, addr):
 		return core.BNGetLiftedILForInstruction(self.handle, arch.handle, addr)
 
@@ -4637,6 +4685,14 @@ class Function(object):
 			display_type = core.BNIntegerDisplayType_by_name[display_type]
 		core.BNSetIntegerConstantDisplayType(self.handle, arch.handle, instr_addr, value, operand, display_type)
 
+	def reanalyze(self):
+		"""
+		``reanalyze`` causes this functions to be reanalyzed. This function does not wait for the analysis to finish.
+
+		:rtype: None
+		"""
+		core.BNReanalyzeFunction(self.handle)
+
 class BasicBlockEdge:
 	def __init__(self, branch_type, target, arch):
 		self.type = core.BNBranchType_names[branch_type]