[SharedCache] Rework metadata serialization to reduce memory overhead

api/MetadataSerializable.hpp is removed in favor of including
core/MetadataSerializable.hpp. Both headers defined types with the same
name leading to One Definition Rule violations and surprising behavior.

The serialization and deserialization context are now created on-demand
during serialization rather than being a member of
`MetadataSerializable`. This reduces the size of every serializable
object by ~220 bytes.

The context is passed explicitly as an argument to `Serialize` /
`Deserialize`. As a result, `Serialize` / `Deserialize` can now be free
functions rather than member functions.

Since `MetadataSerializable` is not used for dynamic dispatch,
the virtual methods are removed and the class is updated to be a class
template using CRTP. This allows delegating to the derived class's
`Load` and `Store` methods without the additional size overhead of the
vtable pointer in every serializable object.

These changes reduce the memory footprint of Binary Ninja after loading
the macOS shared cache and loading a single dylib from it from 8.3GB to
4.6GB.

1 files changed, 395 insertions, 0 deletions

diff --git a/view/sharedcache/api/sharedcache.cpp b/view/sharedcache/api/sharedcache.cpp
index 5ca7c480..45ca6377 100644
--- a/view/sharedcache/api/sharedcache.cpp
+++ b/view/sharedcache/api/sharedcache.cpp
@@ -221,4 +221,399 @@ namespace SharedCacheAPI {
 	{
 		BNDSCFindSymbolAtAddressAndApplyToAddress(m_object, symbolLocation, targetLocation, triggerReanalysis);
 	}
+
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const mach_header_64& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.magic, context.allocator);
+	bArr.PushBack(b.cputype, context.allocator);
+	bArr.PushBack(b.cpusubtype, context.allocator);
+	bArr.PushBack(b.filetype, context.allocator);
+	bArr.PushBack(b.ncmds, context.allocator);
+	bArr.PushBack(b.sizeofcmds, context.allocator);
+	bArr.PushBack(b.flags, context.allocator);
+	bArr.PushBack(b.reserved, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, mach_header_64& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.magic = bArr[0].GetInt64();
+	b.cputype = bArr[1].GetInt64();
+	b.cpusubtype = bArr[2].GetInt64();
+	b.filetype = bArr[3].GetInt64();
+	b.ncmds = bArr[4].GetInt64();
+	b.sizeofcmds = bArr[5].GetInt64();
+	b.flags = bArr[6].GetInt64();
+	b.reserved = bArr[7].GetInt64();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const symtab_command& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.symoff, context.allocator);
+	bArr.PushBack(b.nsyms, context.allocator);
+	bArr.PushBack(b.stroff, context.allocator);
+	bArr.PushBack(b.strsize, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, symtab_command& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.symoff = bArr[2].GetUint();
+	b.nsyms = bArr[3].GetUint();
+	b.stroff = bArr[4].GetUint();
+	b.strsize = bArr[5].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const dysymtab_command& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.ilocalsym, context.allocator);
+	bArr.PushBack(b.nlocalsym, context.allocator);
+	bArr.PushBack(b.iextdefsym, context.allocator);
+	bArr.PushBack(b.nextdefsym, context.allocator);
+	bArr.PushBack(b.iundefsym, context.allocator);
+	bArr.PushBack(b.nundefsym, context.allocator);
+	bArr.PushBack(b.tocoff, context.allocator);
+	bArr.PushBack(b.ntoc, context.allocator);
+	bArr.PushBack(b.modtaboff, context.allocator);
+	bArr.PushBack(b.nmodtab, context.allocator);
+	bArr.PushBack(b.extrefsymoff, context.allocator);
+	bArr.PushBack(b.nextrefsyms, context.allocator);
+	bArr.PushBack(b.indirectsymoff, context.allocator);
+	bArr.PushBack(b.nindirectsyms, context.allocator);
+	bArr.PushBack(b.extreloff, context.allocator);
+	bArr.PushBack(b.nextrel, context.allocator);
+	bArr.PushBack(b.locreloff, context.allocator);
+	bArr.PushBack(b.nlocrel, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, dysymtab_command& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.ilocalsym = bArr[2].GetUint();
+	b.nlocalsym = bArr[3].GetUint();
+	b.iextdefsym = bArr[4].GetUint();
+	b.nextdefsym = bArr[5].GetUint();
+	b.iundefsym = bArr[6].GetUint();
+	b.nundefsym = bArr[7].GetUint();
+	b.tocoff = bArr[8].GetUint();
+	b.ntoc = bArr[9].GetUint();
+	b.modtaboff = bArr[10].GetUint();
+	b.nmodtab = bArr[11].GetUint();
+	b.extrefsymoff = bArr[12].GetUint();
+	b.nextrefsyms = bArr[13].GetUint();
+	b.indirectsymoff = bArr[14].GetUint();
+	b.nindirectsyms = bArr[15].GetUint();
+	b.extreloff = bArr[16].GetUint();
+	b.nextrel = bArr[17].GetUint();
+	b.locreloff = bArr[18].GetUint();
+	b.nlocrel = bArr[19].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const dyld_info_command& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.rebase_off, context.allocator);
+	bArr.PushBack(b.rebase_size, context.allocator);
+	bArr.PushBack(b.bind_off, context.allocator);
+	bArr.PushBack(b.bind_size, context.allocator);
+	bArr.PushBack(b.weak_bind_off, context.allocator);
+	bArr.PushBack(b.weak_bind_size, context.allocator);
+	bArr.PushBack(b.lazy_bind_off, context.allocator);
+	bArr.PushBack(b.lazy_bind_size, context.allocator);
+	bArr.PushBack(b.export_off, context.allocator);
+	bArr.PushBack(b.export_size, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, dyld_info_command& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.rebase_off = bArr[2].GetUint();
+	b.rebase_size = bArr[3].GetUint();
+	b.bind_off = bArr[4].GetUint();
+	b.bind_size = bArr[5].GetUint();
+	b.weak_bind_off = bArr[6].GetUint();
+	b.weak_bind_size = bArr[7].GetUint();
+	b.lazy_bind_off = bArr[8].GetUint();
+	b.lazy_bind_size = bArr[9].GetUint();
+	b.export_off = bArr[10].GetUint();
+	b.export_size = bArr[11].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const routines_command_64& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.init_address, context.allocator);
+	bArr.PushBack(b.init_module, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, routines_command_64& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.init_address = bArr[2].GetUint();
+	b.init_module = bArr[3].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const function_starts_command& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.funcoff, context.allocator);
+	bArr.PushBack(b.funcsize, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, function_starts_command& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.funcoff = bArr[2].GetUint();
+	b.funcsize = bArr[3].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const std::vector<section_64>& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	for (auto& s : b)
+	{
+		rapidjson::Value sArr(rapidjson::kArrayType);
+		std::string sectNameStr;
+		char sectName[16];
+		memcpy(sectName, s.sectname, 16);
+		sectName[15] = 0;
+		sectNameStr = std::string(sectName);
+		sArr.PushBack(rapidjson::Value(sectNameStr.c_str(), context.allocator), context.allocator);
+		std::string segNameStr;
+		char segName[16];
+		memcpy(segName, s.segname, 16);
+		segName[15] = 0;
+		segNameStr = std::string(segName);
+		sArr.PushBack(rapidjson::Value(segNameStr.c_str(), context.allocator), context.allocator);
+		sArr.PushBack(s.addr, context.allocator);
+		sArr.PushBack(s.size, context.allocator);
+		sArr.PushBack(s.offset, context.allocator);
+		sArr.PushBack(s.align, context.allocator);
+		sArr.PushBack(s.reloff, context.allocator);
+		sArr.PushBack(s.nreloc, context.allocator);
+		sArr.PushBack(s.flags, context.allocator);
+		sArr.PushBack(s.reserved1, context.allocator);
+		sArr.PushBack(s.reserved2, context.allocator);
+		sArr.PushBack(s.reserved3, context.allocator);
+		bArr.PushBack(sArr, context.allocator);
+	}
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, std::vector<section_64>& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	for (auto& s : bArr)
+	{
+		section_64 sec;
+		auto s2 = s.GetArray();
+		std::string sectNameStr = s2[0].GetString();
+		memcpy(sec.sectname, sectNameStr.c_str(), sectNameStr.size());
+		std::string segNameStr = s2[1].GetString();
+		memcpy(sec.segname, segNameStr.c_str(), segNameStr.size());
+		sec.addr = s2[2].GetUint64();
+		sec.size = s2[3].GetUint64();
+		sec.offset = s2[4].GetUint();
+		sec.align = s2[5].GetUint();
+		sec.reloff = s2[6].GetUint();
+		sec.nreloc = s2[7].GetUint();
+		sec.flags = s2[8].GetUint();
+		sec.reserved1 = s2[9].GetUint();
+		sec.reserved2 = s2[10].GetUint();
+		sec.reserved3 = s2[11].GetUint();
+		b.push_back(sec);
+	}
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const linkedit_data_command& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.dataoff, context.allocator);
+	bArr.PushBack(b.datasize, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, linkedit_data_command& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.dataoff = bArr[2].GetUint();
+	b.datasize = bArr[3].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const segment_command_64& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	std::string segNameStr;
+	char segName[16];
+	memcpy(segName, b.segname, 16);
+	segName[15] = 0;
+	segNameStr = std::string(segName);
+	bArr.PushBack(rapidjson::Value(segNameStr.c_str(), context.allocator), context.allocator);
+	bArr.PushBack(b.vmaddr, context.allocator);
+	bArr.PushBack(b.vmsize, context.allocator);
+	bArr.PushBack(b.fileoff, context.allocator);
+	bArr.PushBack(b.filesize, context.allocator);
+	bArr.PushBack(b.maxprot, context.allocator);
+	bArr.PushBack(b.initprot, context.allocator);
+	bArr.PushBack(b.nsects, context.allocator);
+	bArr.PushBack(b.flags, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, segment_command_64& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	std::string segNameStr = bArr[0].GetString();
+	memcpy(b.segname, segNameStr.c_str(), segNameStr.size());
+	b.vmaddr = bArr[1].GetUint64();
+	b.vmsize = bArr[2].GetUint64();
+	b.fileoff = bArr[3].GetUint64();
+	b.filesize = bArr[4].GetUint64();
+	b.maxprot = bArr[5].GetUint();
+	b.initprot = bArr[6].GetUint();
+	b.nsects = bArr[7].GetUint();
+	b.flags = bArr[8].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const std::vector<segment_command_64>& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	for (auto& s : b)
+	{
+		rapidjson::Value sArr(rapidjson::kArrayType);
+		std::string segNameStr;
+		char segName[16];
+		memcpy(segName, s.segname, 16);
+		segName[15] = 0;
+		segNameStr = std::string(segName);
+		sArr.PushBack(rapidjson::Value(segNameStr.c_str(), context.allocator), context.allocator);
+		sArr.PushBack(s.vmaddr, context.allocator);
+		sArr.PushBack(s.vmsize, context.allocator);
+		sArr.PushBack(s.fileoff, context.allocator);
+		sArr.PushBack(s.filesize, context.allocator);
+		sArr.PushBack(s.maxprot, context.allocator);
+		sArr.PushBack(s.initprot, context.allocator);
+		sArr.PushBack(s.nsects, context.allocator);
+		sArr.PushBack(s.flags, context.allocator);
+		bArr.PushBack(sArr, context.allocator);
+	}
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, std::vector<segment_command_64>& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	for (auto& s : bArr)
+	{
+		segment_command_64 sec;
+		auto s2 = s.GetArray();
+		std::string segNameStr = s2[0].GetString();
+		memcpy(sec.segname, segNameStr.c_str(), segNameStr.size());
+		sec.vmaddr = s2[1].GetUint64();
+		sec.vmsize = s2[2].GetUint64();
+		sec.fileoff = s2[3].GetUint64();
+		sec.filesize = s2[4].GetUint64();
+		sec.maxprot = s2[5].GetUint();
+		sec.initprot = s2[6].GetUint();
+		sec.nsects = s2[7].GetUint();
+		sec.flags = s2[8].GetUint();
+		b.push_back(sec);
+	}
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const build_version_command& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	bArr.PushBack(b.cmd, context.allocator);
+	bArr.PushBack(b.cmdsize, context.allocator);
+	bArr.PushBack(b.platform, context.allocator);
+	bArr.PushBack(b.minos, context.allocator);
+	bArr.PushBack(b.sdk, context.allocator);
+	bArr.PushBack(b.ntools, context.allocator);
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, build_version_command& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	b.cmd = bArr[0].GetUint();
+	b.cmdsize = bArr[1].GetUint();
+	b.platform = bArr[2].GetUint();
+	b.minos = bArr[3].GetUint();
+	b.sdk = bArr[4].GetUint();
+	b.ntools = bArr[5].GetUint();
+}
+
+void Serialize(SharedCacheCore::SerializationContext& context, std::string_view name, const std::vector<build_tool_version>& b)
+{
+	rapidjson::Value key(name.data(), context.allocator);
+	rapidjson::Value bArr(rapidjson::kArrayType);
+	for (auto& s : b)
+	{
+		rapidjson::Value sArr(rapidjson::kArrayType);
+		sArr.PushBack(s.tool, context.allocator);
+		sArr.PushBack(s.version, context.allocator);
+		bArr.PushBack(sArr, context.allocator);
+	}
+	context.doc.AddMember(key, bArr, context.allocator);
+}
+
+void Deserialize(SharedCacheCore::DeserializationContext& context, std::string_view name, std::vector<build_tool_version>& b)
+{
+	auto bArr = context.doc[name.data()].GetArray();
+	for (auto& s : bArr)
+	{
+		build_tool_version sec;
+		auto s2 = s.GetArray();
+		sec.tool = s2[0].GetUint();
+		sec.version = s2[1].GetUint();
+		b.push_back(sec);
+	}
+}
+
 }	// namespace SharedCacheAPI