Add BinaryView::ParseTypesFromSource for clang+typelibs

7 files changed, 156 insertions, 29 deletions

diff --git a/binaryninjaapi.h b/binaryninjaapi.h
index d0725e5c..a65bff45 100644
--- a/binaryninjaapi.h
+++ b/binaryninjaapi.h
@@ -1737,6 +1737,7 @@ namespace BinaryNinja {
 	struct PossibleValueSet;
 	class Metadata;
 	class Structure;
+	struct TypeParserResult;
 
 	class QueryMetadataException : public std::exception
 	{
@@ -2114,6 +2115,8 @@ namespace BinaryNinja {
 		bool ParseTypeString(const std::string& text, std::map<QualifiedName, Ref<Type>>& types,
 		    std::map<QualifiedName, Ref<Type>>& variables, std::map<QualifiedName, Ref<Type>>& functions,
 		    std::string& errors, const std::set<QualifiedName>& typesAllowRedefinition = {});
+		bool ParseTypesFromSource(const std::string& text, const std::vector<std::string>& options, const std::vector<std::string>& includeDirs, TypeParserResult& result,
+		    std::string& errors, const std::set<QualifiedName>& typesAllowRedefinition = {});
 
 		std::map<QualifiedName, Ref<Type>> GetTypes();
 		std::vector<QualifiedName> GetTypeNames(const std::string& matching = "");
diff --git a/binaryninjacore.h b/binaryninjacore.h
index 4fb254e7..263f885e 100644
--- a/binaryninjacore.h
+++ b/binaryninjacore.h
@@ -21,6 +21,7 @@
 #ifndef __BINARYNINJACORE_H__
 #define __BINARYNINJACORE_H__
 
+#ifndef BN_TYPE_PARSER
 #ifdef __cplusplus
 #include <cstdint>
 #include <cstddef>
@@ -30,11 +31,12 @@
 #include <stddef.h>
 #include <stdlib.h>
 #endif
+#endif
 
 // Current ABI version for linking to the core. This is incremented any time
 // there are changes to the API that affect linking, including new functions,
 // new types, or modifications to existing functions or types.
-#define BN_CURRENT_CORE_ABI_VERSION 20
+#define BN_CURRENT_CORE_ABI_VERSION 21
 
 // Minimum ABI version that is supported for loading of plugins. Plugins that
 // are linked to an ABI version less than this will not be able to load and
@@ -4169,8 +4171,9 @@ extern "C"
 
 	BINARYNINJACOREAPI bool BNParseTypeString(BNBinaryView* view, const char* text, BNQualifiedNameAndType* result,
 	    char** errors, BNQualifiedNameList* typesAllowRedefinition);
-	BINARYNINJACOREAPI bool BNParseTypesString(BNBinaryView* view, const char* text, BNTypeParserResult* result,
-	    char** errors, BNQualifiedNameList* typesAllowRedefinition);
+	BINARYNINJACOREAPI bool BNParseTypesString(BNBinaryView* view, const char* text, const char* const* options, size_t optionCount,
+		const char* const* includeDirs, size_t includeDirCount, BNTypeParserResult* result, char** errors,
+		BNQualifiedNameList* typesAllowRedefinition);
 	BINARYNINJACOREAPI void BNFreeQualifiedNameAndType(BNQualifiedNameAndType* obj);
 	BINARYNINJACOREAPI void BNFreeQualifiedNameAndTypeArray(BNQualifiedNameAndType* obj, size_t count);
 	BINARYNINJACOREAPI char* BNEscapeTypeName(const char* name, BNTokenEscapingType escaping);
diff --git a/binaryview.cpp b/binaryview.cpp
index 21a37bee..35097619 100644
--- a/binaryview.cpp
+++ b/binaryview.cpp
@@ -3039,7 +3039,11 @@ bool BinaryView::ParseTypeString(const string& source, map<QualifiedName, Ref<Ty
 		i++;
 	}
 
-	bool ok = BNParseTypesString(m_object, source.c_str(), &result, &errorStr, &typesList);
+	vector<const char*> options;
+	vector<const char*> includeDirs;
+
+	bool ok = BNParseTypesString(m_object, source.c_str(), options.data(), options.size(),
+		includeDirs.data(), includeDirs.size(), &result, &errorStr, &typesList);
 	if (errorStr)
 	{
 		errors = errorStr;
@@ -3068,6 +3072,75 @@ bool BinaryView::ParseTypeString(const string& source, map<QualifiedName, Ref<Ty
 }
 
 
+bool BinaryView::ParseTypesFromSource(const string& source, const vector<string>& options, const vector<string>& includeDirs,
+	TypeParserResult& result, string& errors, const std::set<QualifiedName>& typesAllowRedefinition)
+{
+	BNQualifiedNameList typesList;
+	typesList.count = typesAllowRedefinition.size();
+	typesList.names = new BNQualifiedName[typesList.count];
+	size_t i = 0;
+	for (auto& type : typesAllowRedefinition)
+	{
+		typesList.names[i] = type.GetAPIObject();
+		i++;
+	}
+
+	vector<const char*> coreOptions;
+	for (auto& option : options)
+		coreOptions.push_back(option.c_str());
+
+	vector<const char*> coreIncludeDirs;
+	for (auto& includeDir : includeDirs)
+		coreIncludeDirs.push_back(includeDir.c_str());
+
+	BNTypeParserResult apiResult;
+	char* errorStr = nullptr;
+
+	bool ok = BNParseTypesString(m_object, source.c_str(), coreOptions.data(), coreOptions.size(),
+		coreIncludeDirs.data(), coreIncludeDirs.size(), &apiResult, &errorStr, &typesList);
+	if (errorStr)
+	{
+		errors = errorStr;
+		BNFreeString(errorStr);
+	}
+	if (!ok)
+		return false;
+
+	result.types.clear();
+	for (size_t j = 0; j < apiResult.typeCount; ++j)
+	{
+		result.types.push_back({
+			QualifiedName::FromAPIObject(&apiResult.types[j].name),
+			new Type(BNNewTypeReference(apiResult.types[j].type)),
+			apiResult.types[j].isUser
+		});
+	}
+
+	result.variables.clear();
+	for (size_t j = 0; j < apiResult.variableCount; ++j)
+	{
+		result.variables.push_back({
+			QualifiedName::FromAPIObject(&apiResult.variables[j].name),
+			new Type(BNNewTypeReference(apiResult.variables[j].type)),
+			apiResult.types[j].isUser
+		});
+	}
+
+	result.functions.clear();
+	for (size_t j = 0; j < apiResult.functionCount; ++j)
+	{
+		result.functions.push_back({
+			QualifiedName::FromAPIObject(&apiResult.functions[j].name),
+			new Type(BNNewTypeReference(apiResult.functions[j].type)),
+			apiResult.types[j].isUser
+		});
+	}
+
+	BNFreeTypeParserResult(&apiResult);
+	return true;
+}
+
+
 map<QualifiedName, Ref<Type>> BinaryView::GetTypes()
 {
 	size_t count;
diff --git a/python/binaryview.py b/python/binaryview.py
index be4eab96..9093bcda 100644
--- a/python/binaryview.py
+++ b/python/binaryview.py
@@ -6190,13 +6190,15 @@ class BinaryView:
 		finally:
 			core.BNFreeQualifiedNameAndType(result)
 
-	def parse_types_from_string(self, text: str) -> '_types.TypeParserResult':
+	def parse_types_from_string(self, text: str, options: Optional[List[str]] = None, include_dirs: Optional[List[str]] = None) -> '_types.TypeParserResult':
 		"""
 		``parse_types_from_string`` parses string containing C into a :py:Class:`TypeParserResult` objects. This API
 		unlike the :py:meth:`Platform.parse_types_from_source` allows the reference of types already defined
 		in the BinaryView.
 
 		:param str text: C source code string of types, variables, and function types, to create
+		:param options: Optional list of string options to be passed into the type parser
+		:param include_dirs: Optional list of header search directories
 		:return: :py:class:`TypeParserResult` (a SyntaxError is thrown on parse error)
 		:rtype: TypeParserResult
 		:Example:
@@ -6209,12 +6211,27 @@ class BinaryView:
 		if not isinstance(text, str):
 			raise ValueError("Source must be a string")
 
+		if options is None:
+			options = []
+		if include_dirs is None:
+			include_dirs = []
+
 		parse = core.BNTypeParserResult()
 		try:
+			options_cpp = (ctypes.c_char_p * len(options))()
+			for (i, s) in enumerate(options):
+				options_cpp[i] = core.cstr(s)
+
+			include_dirs_cpp = (ctypes.c_char_p * len(include_dirs))()
+			for (i, s) in enumerate(include_dirs):
+				include_dirs_cpp[i] = core.cstr(s)
+
 			errors = ctypes.c_char_p()
 			type_list = core.BNQualifiedNameList()
 			type_list.count = 0
-			if not core.BNParseTypesString(self.handle, text, parse, errors, type_list):
+			if not core.BNParseTypesString(
+					self.handle, text, options_cpp, len(options), include_dirs_cpp,
+					len(include_dirs), parse, errors, type_list):
 				assert errors.value is not None, "core.BNParseTypesString returned errors set to None"
 				error_str = errors.value.decode("utf-8")
 				core.free_string(errors)
diff --git a/python/platform.py b/python/platform.py
index fc46882b..05cbaabf 100644
--- a/python/platform.py
+++ b/python/platform.py
@@ -26,6 +26,7 @@ from typing import List, Dict, Optional
 import binaryninja
 from . import _binaryninjacore as core
 from . import types
+from . import typeparser
 from . import callingconvention
 from . import typelibrary
 from . import architecture
@@ -411,8 +412,8 @@ class Platform(metaclass=_PlatformMetaClass):
 		:param include_dirs: optional list of string filename include directories
 		:type include_dirs: list(str)
 		:param str auto_type_source: optional source of types if used for automatically generated types
-		:return: :py:class:`TypeParserResult` (a SyntaxError is thrown on parse error)
-		:rtype: TypeParserResult
+		:return: :py:class:`BasicTypeParserResult` (a SyntaxError is thrown on parse error)
+		:rtype: BasicTypeParserResult
 		:Example:
 
 			>>> platform.parse_types_from_source('int foo;\\nint bar(int x);\\nstruct bas{int x,y;};\\n')
@@ -453,7 +454,7 @@ class Platform(metaclass=_PlatformMetaClass):
 			name = types.QualifiedName._from_core_struct(parse.functions[i].name)
 			functions[name] = types.Type.create(core.BNNewTypeReference(parse.functions[i].type), platform=self)
 		core.BNFreeTypeParserResult(parse)
-		return types.TypeParserResult(type_dict, variables, functions)
+		return typeparser.BasicTypeParserResult(type_dict, variables, functions)
 
 	def parse_types_from_source_file(self, filename, include_dirs: Optional[List[str]] = None, auto_type_source=None):
 		"""
@@ -464,8 +465,8 @@ class Platform(metaclass=_PlatformMetaClass):
 		:param include_dirs: optional list of string filename include directories
 		:type include_dirs: list(str)
 		:param str auto_type_source: optional source of types if used for automatically generated types
-		:return: :py:class:`TypeParserResult` (a SyntaxError is thrown on parse error)
-		:rtype: TypeParserResult
+		:return: :py:class:`BasicTypeParserResult` (a SyntaxError is thrown on parse error)
+		:rtype: BasicTypeParserResult
 		:Example:
 
 			>>> file = "/Users/binja/tmp.c"
@@ -506,7 +507,7 @@ class Platform(metaclass=_PlatformMetaClass):
 			name = types.QualifiedName._from_core_struct(parse.functions[i].name)
 			functions[name] = types.Type.create(core.BNNewTypeReference(parse.functions[i].type), platform=self)
 		core.BNFreeTypeParserResult(parse)
-		return types.TypeParserResult(type_dict, variables, functions)
+		return typeparser.BasicTypeParserResult(type_dict, variables, functions)
 
 	@property
 	def arch(self):
diff --git a/python/typeparser.py b/python/typeparser.py
index f1dffe13..f4eba3cf 100644
--- a/python/typeparser.py
+++ b/python/typeparser.py
@@ -121,6 +121,16 @@ class ParsedType:
 
 
 @dataclasses.dataclass(frozen=True)
+class BasicTypeParserResult:
+	types: Dict['types.QualifiedName', 'types.Type']
+	variables: Dict['types.QualifiedName', 'types.Type']
+	functions: Dict['types.QualifiedName', 'types.Type']
+
+	def __repr__(self):
+		return f"<types: {self.types}, variables: {self.variables}, functions: {self.functions}>"
+
+
+@dataclasses.dataclass(frozen=True)
 class TypeParserResult:
 	types: Dict['types.QualifiedName', ParsedType]
 	variables: Dict['types.QualifiedName', ParsedType]
@@ -392,14 +402,14 @@ class TypeParser(metaclass=_TypeParserMetaclass):
 			return False
 
 	def preprocess_source(
-			self, source: str, file_name: str, platform: platform.Platform,
+			self, source: str, file_name: str, platform: 'platform.Platform',
 			existing_types: Optional[List[QualifiedNameTypeAndId]] = None,
 			options: Optional[List[str]] = None, include_dirs: Optional[List[str]] = None
 	) -> Tuple[Optional[str], List[TypeParserError]]:
 		raise NotImplementedError("Not implemented")
 
 	def parse_types_from_source(
-			self, source: str, file_name: str, platform: platform.Platform,
+			self, source: str, file_name: str, platform: 'platform.Platform',
 			existing_types: Optional[List[QualifiedNameTypeAndId]] = None,
 			options: Optional[List[str]] = None, include_dirs: Optional[List[str]] = None,
 			auto_type_source: str = ""
@@ -407,7 +417,7 @@ class TypeParser(metaclass=_TypeParserMetaclass):
 		raise NotImplementedError("Not implemented")
 
 	def parse_type_string(
-			self, source: str, platform: platform.Platform,
+			self, source: str, platform: 'platform.Platform',
 			existing_types: Optional[List[QualifiedNameTypeAndId]] = None
 	) -> Tuple[Optional[Tuple['types.QualifiedNameType', 'types.Type']], List[TypeParserError]]:
 		raise NotImplementedError("Not implemented")
@@ -416,7 +426,7 @@ class TypeParser(metaclass=_TypeParserMetaclass):
 class CoreTypeParser(TypeParser):
 
 	def preprocess_source(
-			self, source: str, file_name: str, platform: platform.Platform,
+			self, source: str, file_name: str, platform: 'platform.Platform',
 			existing_types: Optional[List[QualifiedNameTypeAndId]] = None,
 			options: Optional[List[str]] = None, include_dirs: Optional[List[str]] = None
 	) -> Tuple[Optional[str], List[TypeParserError]]:
@@ -464,7 +474,7 @@ class CoreTypeParser(TypeParser):
 		return output, errors
 
 	def parse_types_from_source(
-			self, source: str, file_name: str, platform: platform.Platform,
+			self, source: str, file_name: str, platform: 'platform.Platform',
 			existing_types: Optional[List[QualifiedNameTypeAndId]] = None,
 			options: Optional[List[str]] = None, include_dirs: Optional[List[str]] = None,
 			auto_type_source: str = ""
@@ -514,7 +524,7 @@ class CoreTypeParser(TypeParser):
 
 
 	def parse_type_string(
-			self, source: str, platform: platform.Platform,
+			self, source: str, platform: 'platform.Platform',
 			existing_types: Optional[List[QualifiedNameTypeAndId]] = None
 	) -> Tuple[Optional[Tuple['types.QualifiedNameType', 'types.Type']], List[TypeParserError]]:
 		if existing_types is None:
diff --git a/suite/api_test.py b/suite/api_test.py
index 458c3ccb..93f79bd6 100644
--- a/suite/api_test.py
+++ b/suite/api_test.py
@@ -511,6 +511,17 @@ class TypeParserTest(unittest.TestCase):
 	def setUp(self):
 		self.arch = 'x86_64'
 		self.p = Platform[self.arch]
+		self.parser = TypeParser['ClangTypeParser']
+
+	def parse_types_from_source(self, source):
+		(types, errors) = self.parser.parse_types_from_source(source, "types.hpp", self.p)
+		if types is None:
+			raise SyntaxError('\n'.join(str(e) for e in errors))
+		return BasicTypeParserResult(
+			types=dict(zip([t.name for t in types.types], [t.type for t in types.types])),
+			variables=dict(zip([t.name for t in types.variables], [t.type for t in types.variables])),
+			functions=dict(zip([t.name for t in types.functions], [t.type for t in types.functions])),
+		)
 
 	def test_integers(self):
 		integers = [("a", "char a;", 1, True), ("b", "unsigned char b;", 1, False), ("c", "signed char c;", 1, True),
@@ -526,7 +537,7 @@ class TypeParserTest(unittest.TestCase):
 
 		for name, definition, size, signed in integers:
 			with self.subTest():
-				result = self.p.parse_types_from_source(definition)
+				result = self.parse_types_from_source(definition)
 				var = result.variables[name]
 				assert len(var) == size, f"Size for type: {definition} != {size} for arch {self.arch}"
 				assert signed == var.signed, f"Sign for type: {definition} isn't {'signed' if signed else 'unsigned'}"
@@ -537,7 +548,7 @@ class TypeParserTest(unittest.TestCase):
 		              ("c", "struct c { uint64_t x; uint32_t y; };", 16, 8, 2), ]
 		for name, definition, size, alignment, members in structures:
 			with self.subTest():
-				result = self.p.parse_types_from_source(definition)
+				result = self.parse_types_from_source(definition)
 				s = result.types[name]
 				assert len(
 				    s
@@ -584,7 +595,7 @@ class TypeParserTest(unittest.TestCase):
 		              ("a", "struct a { uint8_t a; struct { uint8_t c; uint8_t d; } b; };", 0x3, (0x0, 0x1)), ]
 		for name, definition, size, member_offsets in structures:
 			with self.subTest():
-				result = self.p.parse_types_from_source(definition)
+				result = self.parse_types_from_source(definition)
 				s = result.types[name]
 				assert len(
 				    s
@@ -623,7 +634,7 @@ class TypeParserTest(unittest.TestCase):
 			`another name` (*`third member`)(`another name` `argument name`);
 		};
 		'''
-		types = self.p.parse_types_from_source(valid)
+		types = self.parse_types_from_source(valid)
 		assert types.types['type name with space'] == Type.int(4, False)
 		assert types.types['another name'].name == QualifiedName(
 		    ['type name with space']
@@ -656,8 +667,7 @@ class TypeParserTest(unittest.TestCase):
 			} baz;
 		};
 		'''
-		types = self.p.parse_types_from_source(valid)
-		assert types.types['foo'].type_class == TypeClass.VoidTypeClass
+		types = self.parse_types_from_source(valid)
 		assert types.types['bar'].type_class == TypeClass.StructureTypeClass
 		assert types.types['bar'].type == StructureVariant.ClassStructureType
 		assert types.types['baz'].type_class == TypeClass.StructureTypeClass
@@ -695,7 +705,7 @@ class TypeParserTest(unittest.TestCase):
 		]
 		for source in valid:
 			with self.subTest():
-				types = self.p.parse_types_from_source(source)
+				types = self.parse_types_from_source(source)
 
 
 	def test_parse_empty(self):
@@ -716,7 +726,7 @@ class TypeParserTest(unittest.TestCase):
 		]
 		for source in valid:
 			with self.subTest():
-				types = self.p.parse_types_from_source(source)
+				types = self.parse_types_from_source(source)
 
 		invalid = [
 			# Forward declaration of enum is not allowed
@@ -725,7 +735,7 @@ class TypeParserTest(unittest.TestCase):
 		for source in invalid:
 			with self.subTest():
 				with self.assertRaises(SyntaxError):
-					types = self.p.parse_types_from_source(source)
+					types = self.parse_types_from_source(source)
 
 	def test_parse_nested(self):
 		source = r'''
@@ -752,7 +762,7 @@ class TypeParserTest(unittest.TestCase):
 			} bravo;
 		};
 		'''
-		types = self.p.parse_types_from_source(source)
+		types = self.parse_types_from_source(source)
 		assert types.types['foo'].members[0].type.type_class == TypeClass.EnumerationTypeClass
 		assert types.types['foo'].members[1].type.type_class == TypeClass.StructureTypeClass
 		assert types.types['foo'].members[1].type.type == StructureVariant.StructStructureType
@@ -786,7 +796,7 @@ class TypeParserTest(unittest.TestCase):
 				short DefaultId;
 			};
 		'''
-		types = self.p.parse_types_from_source(source)
+		types = self.parse_types_from_source(source)
 		assert types.types['_LIST_ENTRY'].width == 0x10
 		assert types.types['LIST_ENTRY'].width == 0x10
 		assert types.types['LIST_ENTRY1'].width == 0x10
@@ -2220,6 +2230,16 @@ class TestWithFunction(TestWithBinaryView):
 
 
 class TestBinaryView(TestWithBinaryView):
+	def setUp(self):
+		# Switch to clang by default for these tests, since they rely on it
+		self.original_parser = Settings().get_string("analysis.types.parserName")
+		Settings().set_string("analysis.types.parserName", "ClangTypeParser")
+		super().setUp()
+
+	def tearDown(self) -> None:
+		Settings().set_string("analysis.types.parserName", self.original_parser)
+		super().tearDown()
+
 	def test_TagType(self):
 		tt = self.bv.tag_types["Crashes"]
 		t2 = self.bv.tag_types["Bugs"]