Improve calling convention support for variadic argument resolution

4 files changed, 38 insertions, 2 deletions

diff --git a/binaryninjaapi.h b/binaryninjaapi.h
index 459ddcb5..08e75d49 100644
--- a/binaryninjaapi.h
+++ b/binaryninjaapi.h
@@ -4756,6 +4756,7 @@ __attribute__ ((format (printf, 1, 2)))
 		static void FreeRegisterListCallback(void* ctxt, uint32_t* regs);
 
 		static bool AreArgumentRegistersSharedIndexCallback(void* ctxt);
+		static bool AreArgumentRegistersUsedForVarArgsCallback(void* ctxt);
 		static bool IsStackReservedForArgumentRegistersCallback(void* ctxt);
 		static bool IsStackAdjustedOnReturnCallback(void* ctxt);
 		static bool IsEligibleForHeuristicsCallback(void* ctxt);
@@ -4784,6 +4785,7 @@ __attribute__ ((format (printf, 1, 2)))
 		virtual std::vector<uint32_t> GetIntegerArgumentRegisters();
 		virtual std::vector<uint32_t> GetFloatArgumentRegisters();
 		virtual bool AreArgumentRegistersSharedIndex();
+		virtual bool AreArgumentRegistersUsedForVarArgs();
 		virtual bool IsStackReservedForArgumentRegisters();
 		virtual bool IsStackAdjustedOnReturn();
 		virtual bool IsEligibleForHeuristics();
@@ -4812,6 +4814,7 @@ __attribute__ ((format (printf, 1, 2)))
 		virtual std::vector<uint32_t> GetIntegerArgumentRegisters() override;
 		virtual std::vector<uint32_t> GetFloatArgumentRegisters() override;
 		virtual bool AreArgumentRegistersSharedIndex() override;
+		virtual bool AreArgumentRegistersUsedForVarArgs() override;
 		virtual bool IsStackReservedForArgumentRegisters() override;
 		virtual bool IsStackAdjustedOnReturn() override;
 		virtual bool IsEligibleForHeuristics() override;
diff --git a/binaryninjacore.h b/binaryninjacore.h
index 71cc538d..35f653a2 100644
--- a/binaryninjacore.h
+++ b/binaryninjacore.h
@@ -28,14 +28,14 @@
 // Current ABI version for linking to the core. This is incremented any time
 // there are changes to the API that affect linking, including new functions,
 // new types, or modifications to existing functions or types.
-#define BN_CURRENT_CORE_ABI_VERSION 9
+#define BN_CURRENT_CORE_ABI_VERSION 10
 
 // Minimum ABI version that is supported for loading of plugins. Plugins that
 // are linked to an ABI version less than this will not be able to load and
 // will require rebuilding. The minimum version is increased when there are
 // incompatible changes that break binary compatibility, such as changes to
 // existing types or functions.
-#define BN_MINIMUM_CORE_ABI_VERSION 7
+#define BN_MINIMUM_CORE_ABI_VERSION 10
 
 #ifdef __GNUC__
 #  ifdef BINARYNINJACORE_LIBRARY
@@ -2057,6 +2057,8 @@ extern "C"
 			BNFunction* func, BNVariable* result);
 		void (*getParameterVariableForIncomingVariable)(void* ctxt, const BNVariable* var,
 			BNFunction* func, BNVariable* result);
+
+		bool (*areArgumentRegistersUsedForVarArgs)(void* ctxt);
 	};
 
 	struct BNVariableNameAndType
@@ -4661,6 +4663,7 @@ __attribute__ ((format (printf, 1, 2)))
 	BINARYNINJACOREAPI uint32_t* BNGetIntegerArgumentRegisters(BNCallingConvention* cc, size_t* count);
 	BINARYNINJACOREAPI uint32_t* BNGetFloatArgumentRegisters(BNCallingConvention* cc, size_t* count);
 	BINARYNINJACOREAPI bool BNAreArgumentRegistersSharedIndex(BNCallingConvention* cc);
+	BINARYNINJACOREAPI bool BNAreArgumentRegistersUsedForVarArgs(BNCallingConvention* cc);
 	BINARYNINJACOREAPI bool BNIsStackReservedForArgumentRegisters(BNCallingConvention* cc);
 	BINARYNINJACOREAPI bool BNIsStackAdjustedOnReturn(BNCallingConvention* cc);
 	BINARYNINJACOREAPI bool BNIsEligibleForHeuristics(BNCallingConvention* cc);
diff --git a/callingconvention.cpp b/callingconvention.cpp
index b6354381..33fe45ff 100644
--- a/callingconvention.cpp
+++ b/callingconvention.cpp
@@ -41,6 +41,7 @@ CallingConvention::CallingConvention(Architecture* arch, const string& name)
 	cc.getFloatArgumentRegisters = GetFloatArgumentRegistersCallback;
 	cc.freeRegisterList = FreeRegisterListCallback;
 	cc.areArgumentRegistersSharedIndex = AreArgumentRegistersSharedIndexCallback;
+	cc.areArgumentRegistersUsedForVarArgs = AreArgumentRegistersUsedForVarArgsCallback;
 	cc.isStackReservedForArgumentRegisters = IsStackReservedForArgumentRegistersCallback;
 	cc.isStackAdjustedOnReturn = IsStackAdjustedOnReturnCallback;
 	cc.isEligibleForHeuristics = IsEligibleForHeuristicsCallback;
@@ -131,6 +132,13 @@ bool CallingConvention::AreArgumentRegistersSharedIndexCallback(void* ctxt)
 }
 
 
+bool CallingConvention::AreArgumentRegistersUsedForVarArgsCallback(void* ctxt)
+{
+	CallingConvention* cc = (CallingConvention*)ctxt;
+	return cc->AreArgumentRegistersUsedForVarArgs();
+}
+
+
 bool CallingConvention::IsStackReservedForArgumentRegistersCallback(void* ctxt)
 {
 	CallingConvention* cc = (CallingConvention*)ctxt;
@@ -280,6 +288,12 @@ bool CallingConvention::AreArgumentRegistersSharedIndex()
 }
 
 
+bool CallingConvention::AreArgumentRegistersUsedForVarArgs()
+{
+	return true;
+}
+
+
 bool CallingConvention::IsStackReservedForArgumentRegisters()
 {
 	return false;
@@ -409,6 +423,12 @@ bool CoreCallingConvention::AreArgumentRegistersSharedIndex()
 }
 
 
+bool CoreCallingConvention::AreArgumentRegistersUsedForVarArgs()
+{
+	return BNAreArgumentRegistersUsedForVarArgs(m_object);
+}
+
+
 bool CoreCallingConvention::IsStackReservedForArgumentRegisters()
 {
 	return BNIsStackReservedForArgumentRegisters(m_object);
diff --git a/python/callingconvention.py b/python/callingconvention.py
index aaf86239..c29404aa 100644
--- a/python/callingconvention.py
+++ b/python/callingconvention.py
@@ -38,6 +38,7 @@ class CallingConvention(object):
 	int_arg_regs = []
 	float_arg_regs = []
 	arg_regs_share_index = False
+	arg_regs_for_varargs = True
 	stack_reserved_for_arg_regs = False
 	stack_adjusted_on_return = False
 	eligible_for_heuristics = True
@@ -64,6 +65,7 @@ class CallingConvention(object):
 			self._cb.getFloatArgumentRegisters = self._cb.getFloatArgumentRegisters.__class__(self._get_float_arg_regs)
 			self._cb.freeRegisterList = self._cb.freeRegisterList.__class__(self._free_register_list)
 			self._cb.areArgumentRegistersSharedIndex = self._cb.areArgumentRegistersSharedIndex.__class__(self._arg_regs_share_index)
+			self._cb.areArgumentRegistersUsedForVarArgs = self._cb.areArgumentRegistersUsedForVarArgs.__class__(self._arg_regs_used_for_varargs)
 			self._cb.isStackReservedForArgumentRegisters = self._cb.isStackReservedForArgumentRegisters.__class__(self._stack_reserved_for_arg_regs)
 			self._cb.isStackAdjustedOnReturn = self._cb.isStackAdjustedOnReturn.__class__(self._stack_adjusted_on_return)
 			self._cb.isEligibleForHeuristics = self._cb.isEligibleForHeuristics.__class__(self._eligible_for_heuristics)
@@ -83,6 +85,7 @@ class CallingConvention(object):
 			self.arch = binaryninja.architecture.CoreArchitecture._from_cache(core.BNGetCallingConventionArchitecture(self.handle))
 			self.__dict__["name"] = core.BNGetCallingConventionName(self.handle)
 			self.__dict__["arg_regs_share_index"] = core.BNAreArgumentRegistersSharedIndex(self.handle)
+			self.__dict__["arg_regs_for_varargs"] = core.BNAreArgumentRegistersUsedForVarArgs(self.handle)
 			self.__dict__["stack_reserved_for_arg_regs"] = core.BNIsStackReservedForArgumentRegisters(self.handle)
 			self.__dict__["stack_adjusted_on_return"] = core.BNIsStackAdjustedOnReturn(self.handle)
 			self.__dict__["eligible_for_heuristics"] = core.BNIsEligibleForHeuristics(self.handle)
@@ -257,6 +260,13 @@ class CallingConvention(object):
 			log.log_error(traceback.format_exc())
 			return False
 
+	def _arg_regs_used_for_varargs(self, ctxt):
+		try:
+			return self.__class__.arg_regs_for_varargs
+		except:
+			log.log_error(traceback.format_exc())
+			return False
+
 	def _stack_reserved_for_arg_regs(self, ctxt):
 		try:
 			return self.__class__.stack_reserved_for_arg_regs