better handling for when utf8 decoding fails

6 files changed, 24 insertions, 6 deletions

diff --git a/python/architecture.py b/python/architecture.py
index 95af9ddd..0b149057 100644
--- a/python/architecture.py
+++ b/python/architecture.py
@@ -3224,7 +3224,10 @@ class InstructionTextToken:
 			token_type = InstructionTextTokenType(tokens[j].type)
 			text = tokens[j].text
 			if not isinstance(text, str):
-				text = text.decode("utf-8")
+				try:
+					text = text.decode("utf-8")
+				except UnicodeDecodeError:
+					text = text.decode("charmap")
 			width = tokens[j].width
 			value = tokens[j].value
 			size = tokens[j].size
diff --git a/python/compatibility.py b/python/compatibility.py
index 88abb2f8..5632c7a5 100644
--- a/python/compatibility.py
+++ b/python/compatibility.py
@@ -25,7 +25,10 @@ def pyNativeStr(arg):
 	if isinstance(arg, str):
 		return arg
 	else:
-		return arg.decode('utf8')
+		try:
+			return arg.decode('utf8')
+		except UnicodeDecodeError:
+			return arg.decode('charmap')
 
 
 def valid_import(mod_name):
diff --git a/python/databuffer.py b/python/databuffer.py
index a54a6cc0..55b0f782 100644
--- a/python/databuffer.py
+++ b/python/databuffer.py
@@ -128,7 +128,10 @@ class DataBuffer:
 		data = core.BNGetDataBufferContents(self.handle)
 		assert data is not None, "core.BNGetDataBufferContents returned None"
 		ctypes.memmove(buf, data, len(self))
-		return buf.raw.decode('utf8')
+		try:
+			return buf.raw.decode('utf8')
+		except UnicodeDecodeError:
+			return buf.raw.decode('charmap')
 
 	def __bytes__(self):
 		buf = ctypes.create_string_buffer(len(self))
diff --git a/python/demangle.py b/python/demangle.py
index 714f76cc..80916962 100644
--- a/python/demangle.py
+++ b/python/demangle.py
@@ -131,7 +131,10 @@ def demangle_llvm(mangled_name: str, options: Optional[Union[bool, binaryview.Bi
 	)
 	):
 		for i in range(outSize.value):
-			names.append(outName[i].decode('utf8'))  # type: ignore
+			try:
+				names.append(outName[i].decode('utf8'))  # type: ignore
+			except UnicodeDecodeError:
+				names.append(outName[i].decode('charmap'))  # type: ignore
 		core.BNFreeDemangledName(ctypes.byref(outName), outSize.value)
 		return names
 	return None
diff --git a/python/generator.cpp b/python/generator.cpp
index 3fdd2fe4..2e8fd009 100644
--- a/python/generator.cpp
+++ b/python/generator.cpp
@@ -287,7 +287,10 @@ int main(int argc, char* argv[])
 	fprintf(out, "	if isinstance(arg, str):\n");
 	fprintf(out, "		return arg\n");
 	fprintf(out, "	else:\n");
-	fprintf(out, "		return arg.decode('utf8')\n\n\n");
+	fprintf(out, "		try:\n");
+	fprintf(out, "			return arg.decode('utf8')\n");
+	fprintf(out, "		except UnicodeDecodeError:\n");
+	fprintf(out, "			return arg.decode('charmap')\n\n\n");
 
 	fprintf(out, "def free_string(value:ctypes.c_char_p) -> None:\n");
 	fprintf(out, "	BNFreeString(ctypes.cast(value, ctypes.POINTER(ctypes.c_byte)))\n\n");
diff --git a/python/settings.py b/python/settings.py
index 947d2963..4f63c76f 100644
--- a/python/settings.py
+++ b/python/settings.py
@@ -276,7 +276,10 @@ class Settings:
 		assert result is not None, "core.BNSettingsKeysList returned None"
 		out_list = []
 		for i in range(length.value):
-			out_list.append(result[i].decode('utf8'))
+			try:
+				out_list.append(result[i].decode('utf8'))
+			except UnicodeDecodeError:
+				out_list.append(result[i].decode('charmap'))
 		core.BNFreeStringList(result, length)
 		return out_list