diff options
| author | kat <kat@vector35.com> | 2025-03-19 09:12:52 -0400 |
|---|---|---|
| committer | kat <kat@vector35.com> | 2025-03-19 15:04:23 -0400 |
| commit | 7f3f394c8bb50c987a5237bc74aa503486571058 (patch) | |
| tree | 9f5ada34f6226562b79f0dce70bfebab124a1bdb | |
| parent | e6d4d38e2a5dc66d3c8004cc917bc08e39337482 (diff) | |
Add iOS/macOS MH_FILESET KernelCache View and loader.
This loader is inspired by/based on our dyld_shared_cache loader, following the same design language. It targets primarily the latest kernels, but should support any with the MH_FILESET format.
It allows you to decide which images you would like to map in (the kernel itself included), resulting in targeted analysis when you may only need to load a singular image.
It also supports dropping in compressed KernelCaches, directly from the ipsw.
This is an early solution and we have many more changes and improvements planned. We look forward to your feedback
57 files changed, 16761 insertions, 0 deletions
diff --git a/view/kernelcache/.clang-format b/view/kernelcache/.clang-format new file mode 100644 index 00000000..ba1ac018 --- /dev/null +++ b/view/kernelcache/.clang-format @@ -0,0 +1,86 @@ +--- +AccessModifierOffset: -4 +AlignAfterOpenBracket: DontAlign +AlignConsecutiveMacros: 'true' +AlignConsecutiveAssignments: 'false' +AlignConsecutiveDeclarations: 'false' +AlignEscapedNewlines: DontAlign +AlignOperands: 'false' +AlignTrailingComments: 'true' +AllowAllArgumentsOnNextLine: 'true' +AllowAllConstructorInitializersOnNextLine: 'false' +AllowAllParametersOfDeclarationOnNextLine: 'false' +AllowShortBlocksOnASingleLine: 'false' +AllowShortCaseLabelsOnASingleLine: 'false' +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: Inline +AllowShortLoopsOnASingleLine: 'false' +AlwaysBreakAfterDefinitionReturnType: None +AlwaysBreakAfterReturnType: None +AlwaysBreakBeforeMultilineStrings: 'true' +AlwaysBreakTemplateDeclarations: 'Yes' +BinPackArguments: 'true' +BinPackParameters: 'true' +BreakBeforeBraces: Custom +BraceWrapping: + AfterCaseLabel: 'true' + AfterClass: 'true' + AfterControlStatement: 'true' + AfterEnum: 'true' + AfterFunction: 'true' + AfterNamespace: 'false' + AfterObjCDeclaration: 'true' + AfterStruct: 'true' + AfterUnion: 'true' + AfterExternBlock: 'true' + BeforeCatch: 'true' + BeforeElse: 'true' + IndentBraces: 'false' + SplitEmptyFunction: 'false' + SplitEmptyRecord: 'false' + SplitEmptyNamespace: 'false' +BreakBeforeBinaryOperators: NonAssignment +BreakBeforeTernaryOperators: 'false' +BreakConstructorInitializers: AfterColon +BreakInheritanceList: AfterColon +ColumnLimit: '120' +CompactNamespaces: 'true' +ConstructorInitializerAllOnOneLineOrOnePerLine: 'false' +Cpp11BracedListStyle: 'true' +DerivePointerAlignment: 'false' +DisableFormat: 'false' +EmptyLineBeforeAccessModifier: Always +FixNamespaceComments: 'true' +IncludeBlocks: Preserve +IndentCaseLabels: 'false' +IndentPPDirectives: BeforeHash +IndentWrappedFunctionNames: 'true' +KeepEmptyLinesAtTheStartOfBlocks: 'false' +MaxEmptyLinesToKeep: '2' +NamespaceIndentation: All +PenaltyIndentedWhitespace: 100 +PointerAlignment: Left +SortIncludes: 'false' +SortUsingDeclarations: 'true' +SpaceAfterCStyleCast: 'false' +SpaceAfterLogicalNot: 'false' +SpaceAfterTemplateKeyword: 'true' +SpaceBeforeAssignmentOperators: 'true' +SpaceBeforeCpp11BracedList: 'true' +SpaceBeforeCtorInitializerColon: 'true' +SpaceBeforeInheritanceColon: 'true' +SpaceBeforeParens: ControlStatements +SpaceBeforeRangeBasedForLoopColon: 'true' +SpaceInEmptyParentheses: 'false' +SpacesBeforeTrailingComments: '2' +SpacesInAngles: 'false' +SpacesInCStyleCastParentheses: 'false' +SpacesInContainerLiterals: 'false' +SpacesInParentheses: 'false' +SpacesInSquareBrackets: 'false' +IndentWidth: '4' +TabWidth: '4' +UseTab: Always + +... diff --git a/view/kernelcache/CMakeLists.txt b/view/kernelcache/CMakeLists.txt new file mode 100644 index 00000000..ee16db6e --- /dev/null +++ b/view/kernelcache/CMakeLists.txt @@ -0,0 +1,111 @@ +cmake_minimum_required(VERSION 3.13 FATAL_ERROR) + +project(kernelcache) + +if(NOT BN_INTERNAL_BUILD) + find_path( + BN_API_PATH + NAMES binaryninjaapi.h + HINTS ../.. binaryninjaapi $ENV{BN_API_PATH} + REQUIRED + ) + add_subdirectory(${BN_API_PATH} binaryninjaapi) +endif() + +if (NOT BN_INTERNAL_BUILD) + if(WIN32) + set(MSVC_VERSION msvc2022_64 CACHE STRING "Version of MSVC Qt is built with" ) + endif() + set(QT_VERSION 6.7.2 CACHE STRING "Version of Qt to use") + + if(NOT CMAKE_PREFIX_PATH) + if(APPLE) + set(CMAKE_PREFIX_PATH $ENV{HOME}/Qt/${QT_VERSION}/clang_64/lib/cmake) + elseif(WIN32) + set(CMAKE_PREFIX_PATH $ENV{HOMEDRIVE}$ENV{HOMEPATH}/Qt/${QT_VERSION}/${MSVC_VERSION}/lib/cmake) + else() + set(CMAKE_PREFIX_PATH $ENV{HOME}/Qt/${QT_VERSION}/gcc_64/lib/cmake) + endif() + endif() + message("CMAKE_PREFIX_PATH is: ${CMAKE_PREFIX_PATH}") +endif() + +set(HARD_FAIL_MODE OFF CACHE BOOL "Enable hard fail mode") +set(SLIDEINFO_DEBUG_TAGS OFF CACHE BOOL "Enable debug tags in slideinfo") +set(KC_VIEW_NAME "KCView" CACHE STRING "Name of the view") +set(METADATA_VERSION 3 CACHE STRING "Version of the metadata") + +add_subdirectory(core) +add_subdirectory(api) +# add_subdirectory(workflow) + +add_library(kernelcache SHARED + HeadlessPlugin.cpp) + + +if(BN_INTERNAL_BUILD) + set_target_properties(kernelcache PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${BN_CORE_PLUGIN_DIR} + RUNTIME_OUTPUT_DIRECTORY ${BN_CORE_PLUGIN_DIR}) +else() + set_target_properties(kernelcache PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/out/plugins + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/out/plugins + ) +endif() + +set_target_properties(kernelcache PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + POSITION_INDEPENDENT_CODE ON + ) + +target_include_directories(kernelcache PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/core ${CMAKE_CURRENT_SOURCE_DIR}/api ${CMAKE_CURRENT_SOURCE_DIR}/workflow) + +target_link_libraries(kernelcache PUBLIC kernelcacheapi binaryninjaapi kernelcachecore) # kernelcacheworkflow) + + +set(COMPILE_DEFS "") + +if (HARD_FAIL_MODE) + set(COMPILE_DEFS "${COMPILE_DEFS} ABORT_FAILURES;") +endif() + +if (BN_REF_COUNT_DEBUG) + set(COMPILE_DEFS "${COMPILE_DEFS} BN_REF_COUNT_DEBUG;") +endif() + +if (SLIDEINFO_DEBUG_TAGS) + set(COMPILE_DEFS "${COMPILE_DEFS} SLIDEINFO_DEBUG_TAGS;") +endif() + +if (METADATA_VERSION) + set(COMPILE_DEFS "${COMPILE_DEFS} METADATA_VERSION=${METADATA_VERSION};") +else() + message(FATAL_ERROR "No metadata version provided. Fatal.") +endif() + +if (KC_VIEW_NAME) + set(COMPILE_DEFS "${COMPILE_DEFS} KC_VIEW_NAME=\"${KC_VIEW_NAME}\";") +else() + message(FATAL_ERROR "No view name provided. Fatal.") +endif() + +target_compile_definitions(kernelcache PRIVATE ${COMPILE_DEFS}) + +if(NOT HEADLESS) + add_subdirectory(ui) +endif() + +message(" +Apple KernelCache Extraction Plugin + +Metadata Version: ${METADATA_VERSION} +CMAKE_PREFIX_PATH: ${CMAKE_PREFIX_PATH} +Qt Version: ${QT_VERSION} +Crash on Failure: ${HARD_FAIL_MODE} +Slideinfo Debug Tags: ${SLIDEINFO_DEBUG_TAGS} +REFCOUNT_DEBUG: ${BN_REF_COUNT_DEBUG} +") diff --git a/view/kernelcache/HeadlessPlugin.cpp b/view/kernelcache/HeadlessPlugin.cpp new file mode 100644 index 00000000..d14ccd6d --- /dev/null +++ b/view/kernelcache/HeadlessPlugin.cpp @@ -0,0 +1,23 @@ +#include <binaryninjaapi.h> +#include "KCView.h" +#include "KernelCache.h" + +#ifdef __cplusplus +extern "C" { +#endif + // extern void RegisterSharedCacheWorkflow(); +#ifdef __cplusplus +} +#endif + +extern "C" +{ + BN_DECLARE_CORE_ABI_VERSION + + BINARYNINJAPLUGIN bool CorePluginInit() + { + InitKernelcache(); + // RegisterSharedCacheWorkflow(); + return true; + } +}
\ No newline at end of file diff --git a/view/kernelcache/api/CMakeLists.txt b/view/kernelcache/api/CMakeLists.txt new file mode 100644 index 00000000..ae7e454a --- /dev/null +++ b/view/kernelcache/api/CMakeLists.txt @@ -0,0 +1,79 @@ +cmake_minimum_required(VERSION 3.13 FATAL_ERROR) + +project(kernelcacheapi) +file(GLOB BN_MACHO_API_SOURCES *.cpp *.h) +add_library(kernelcacheapi OBJECT ${BN_MACHO_API_SOURCES}) + + +set(COMPILE_DEFS "") + +if (HARD_FAIL_MODE) + set(COMPILE_DEFS "${COMPILE_DEFS} ABORT_FAILURES;") +endif() + +if (BN_REF_COUNT_DEBUG) + set(COMPILE_DEFS "${COMPILE_DEFS} BN_REF_COUNT_DEBUG;") +endif() + +if (SLIDEINFO_DEBUG_TAGS) + set(COMPILE_DEFS "${COMPILE_DEFS} SLIDEINFO_DEBUG_TAGS;") +endif() + +if (METADATA_VERSION) + set(COMPILE_DEFS "${COMPILE_DEFS} METADATA_VERSION=${METADATA_VERSION};") +else() + message(FATAL_ERROR "No metadata version provided. Fatal.") +endif() + +if (KC_VIEW_NAME) + set(COMPILE_DEFS "${COMPILE_DEFS} KC_VIEW_NAME=\"${KC_VIEW_NAME}\";") +else() + message(FATAL_ERROR "No view name provided. Fatal.") +endif() + +target_compile_definitions(kernelcacheapi PRIVATE ${COMPILE_DEFS}) + + +function(get_recursive_include_dirs target result) + # Initialize an empty list to store include directories + set(include_dirs "") + + # Get the include directories of the current target + get_target_property(current_target_includes ${target} INTERFACE_INCLUDE_DIRECTORIES) + if(current_target_includes) + list(APPEND include_dirs ${current_target_includes}) + endif() + + # Get the libraries that this target links to + get_target_property(linked_libraries ${target} INTERFACE_LINK_LIBRARIES) + if(linked_libraries) + foreach(linked_library IN LISTS linked_libraries) + # Skip plain library names (non-target libraries) + if(TARGET ${linked_library}) + # Recursively get include directories from linked libraries + get_recursive_include_dirs(${linked_library} linked_library_includes) + list(APPEND include_dirs ${linked_library_includes}) + endif() + endforeach() + endif() + + # Set the result to the collected include directories + set(${result} ${include_dirs} PARENT_SCOPE) +endfunction() + +get_recursive_include_dirs(binaryninjaapi INCLUDES) + +target_include_directories(kernelcacheapi + PUBLIC ${PROJECT_SOURCE_DIR} ${INCLUDES}) + +set_target_properties(kernelcacheapi PROPERTIES + CXX_STANDARD 17 + CXX_VISIBILITY_PRESET hidden + CXX_STANDARD_REQUIRED ON + VISIBILITY_INLINES_HIDDEN ON + POSITION_INDEPENDENT_CODE ON + ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/out) + +if (NOT DEMO) + add_subdirectory(python) +endif()
\ No newline at end of file diff --git a/view/kernelcache/api/kernelcache.cpp b/view/kernelcache/api/kernelcache.cpp new file mode 100644 index 00000000..6cca9033 --- /dev/null +++ b/view/kernelcache/api/kernelcache.cpp @@ -0,0 +1,191 @@ +// +// Created by kat on 5/21/23. +// + +#include "kernelcacheapi.h" + +namespace KernelCacheAPI { + + KernelCache::KernelCache(Ref<BinaryView> view) { + m_object = BNGetKernelCache(view->GetObject()); + } + + BNKCViewLoadProgress KernelCache::GetLoadProgress(Ref<BinaryView> view) + { + return BNKCViewGetLoadProgress(view->GetFile()->GetSessionId()); + } + + uint64_t KernelCache::FastGetImageCount(Ref<BinaryView> view) + { + return BNKCViewFastGetImageCount(view->GetObject()); + } + + bool KernelCache::LoadImageWithInstallName(std::string installName) + { + char* str = BNAllocString(installName.c_str()); + return BNKCViewLoadImageWithInstallName(m_object, str); + } + + bool KernelCache::LoadImageContainingAddress(uint64_t addr) + { + return BNKCViewLoadImageContainingAddress(m_object, addr); + } + + std::vector<std::string> KernelCache::GetAvailableImages() + { + size_t count; + char** value = BNKCViewGetInstallNames(m_object, &count); + if (value == nullptr) + { + return {}; + } + + std::vector<std::string> result; + for (size_t i = 0; i < count; i++) + { + result.push_back(value[i]); + } + + BNFreeStringList(value, count); + return result; + } + + std::vector<KCImage> KernelCache::GetImages() + { + size_t count; + BNKCImage* value = BNKCViewGetAllImages(m_object, &count); + if (value == nullptr) + { + return {}; + } + + std::vector<KCImage> result; + for (size_t i = 0; i < count; i++) + { + KCImage img; + img.name = value[i].name; + img.headerFileAddress = value[i].headerFileAddress; + for (size_t j = 0; j < value[i].mappingCount; j++) + { + KCImageMemoryMapping mapping; + mapping.name = value[i].mappings[j].name; + mapping.vmAddress = value[i].mappings[j].vmAddress; + mapping.rawViewOffset = value[i].mappings[j].rawViewOffset; + mapping.size = value[i].mappings[j].size; + mapping.loaded = value[i].mappings[j].loaded; + img.mappings.push_back(mapping); + } + result.push_back(img); + } + + BNKCViewFreeAllImages(value, count); + return result; + } + + std::vector<KCImage> KernelCache::GetLoadedImages() + { + size_t count; + auto images = BNKCViewGetLoadedImages(m_object, &count); + if (images == nullptr) + { + return {}; + } + std::vector<KCImage> result; + for (size_t i = 0; i < count; i++) + { + KCImage img; + img.name = images[i].name; + img.headerFileAddress = images[i].headerFileAddress; + for (size_t j = 0; j < images[i].mappingCount; j++) + { + KCImageMemoryMapping mapping; + mapping.name = images[i].mappings[j].name; + mapping.vmAddress = images[i].mappings[j].vmAddress; + mapping.rawViewOffset = images[i].mappings[j].rawViewOffset; + mapping.size = images[i].mappings[j].size; + mapping.loaded = images[i].mappings[j].loaded; + img.mappings.push_back(mapping); + } + result.push_back(img); + } + BNKCViewFreeAllImages(images, count); + + return result; + } + + std::vector<KCSymbol> KernelCache::LoadAllSymbolsAndWait() + { + size_t count; + BNKCSymbolRep* value = BNKCViewLoadAllSymbolsAndWait(m_object, &count); + if (value == nullptr) + { + return {}; + } + + std::vector<KCSymbol> result; + for (size_t i = 0; i < count; i++) + { + KCSymbol sym; + sym.address = value[i].address; + sym.name = value[i].name; + sym.image = value[i].image; + result.push_back(sym); + } + + BNKCViewFreeSymbols(value, count); + return result; + } + + std::string KernelCache::GetNameForAddress(uint64_t address) + { + char* name = BNKCViewGetNameForAddress(m_object, address); + if (name == nullptr) + return {}; + std::string result = name; + BNFreeString(name); + return result; + } + + std::string KernelCache::GetImageNameForAddress(uint64_t address) + { + char* name = BNKCViewGetImageNameForAddress(m_object, address); + if (name == nullptr) + return {}; + std::string result = name; + BNFreeString(name); + return result; + } + + std::optional<KernelCacheMachOHeader> KernelCache::GetMachOHeaderForImage(std::string name) + { + char* str = BNAllocString(name.c_str()); + char* outputStr = BNKCViewGetImageHeaderForName(m_object, str); + if (outputStr == nullptr) + return {}; + std::string output = outputStr; + BNFreeString(outputStr); + if (output.empty()) + return {}; + KernelCacheMachOHeader header = KernelCacheMachOHeader::LoadFromString(output); + return header; + } + + std::optional<KernelCacheMachOHeader> KernelCache::GetMachOHeaderForAddress(uint64_t address) + { + char* outputStr = BNKCViewGetImageHeaderForAddress(m_object, address); + if (outputStr == nullptr) + return {}; + std::string output = outputStr; + BNFreeString(outputStr); + if (output.empty()) + return {}; + KernelCacheMachOHeader header = KernelCacheMachOHeader::LoadFromString(output); + return header; + } + + BNKCViewState KernelCache::GetState() + { + return BNKCViewGetState(m_object); + } + +} // namespace KernelCacheAPI diff --git a/view/kernelcache/api/kernelcacheapi.h b/view/kernelcache/api/kernelcacheapi.h new file mode 100644 index 00000000..149f39bf --- /dev/null +++ b/view/kernelcache/api/kernelcacheapi.h @@ -0,0 +1,272 @@ +#pragma once + +#include <binaryninjaapi.h> +#include "../core/MetadataSerializable.hpp" +#include "../api/view/macho/machoview.h" +#include "kernelcachecore.h" + +using namespace BinaryNinja; + +namespace KernelCacheAPI { + template<class T> + class KCRefCountObject { + void AddRefInternal() { m_refs.fetch_add(1); } + + void ReleaseInternal() { + if (m_refs.fetch_sub(1) == 1) + delete this; + } + + public: + std::atomic<int> m_refs; + T *m_object; + + KCRefCountObject() : m_refs(0), m_object(nullptr) {} + + virtual ~KCRefCountObject() {} + + T *GetObject() const { return m_object; } + + static T *GetObject(KCRefCountObject *obj) { + if (!obj) + return nullptr; + return obj->GetObject(); + } + + void AddRef() { AddRefInternal(); } + + void Release() { ReleaseInternal(); } + + void AddRefForRegistration() { AddRefInternal(); } + }; + + + template<class T, T *(*AddObjectReference)(T *), void (*FreeObjectReference)(T *)> + class KCCoreRefCountObject { + void AddRefInternal() { m_refs.fetch_add(1); } + + void ReleaseInternal() { + if (m_refs.fetch_sub(1) == 1) { + if (!m_registeredRef) + delete this; + } + } + + public: + std::atomic<int> m_refs; + bool m_registeredRef = false; + T *m_object; + + KCCoreRefCountObject() : m_refs(0), m_object(nullptr) {} + + virtual ~KCCoreRefCountObject() {} + + T *GetObject() const { return m_object; } + + static T *GetObject(KCCoreRefCountObject *obj) { + if (!obj) + return nullptr; + return obj->GetObject(); + } + + void AddRef() { + if (m_object && (m_refs != 0)) + AddObjectReference(m_object); + AddRefInternal(); + } + + void Release() { + if (m_object) + FreeObjectReference(m_object); + ReleaseInternal(); + } + + void AddRefForRegistration() { m_registeredRef = true; } + + void ReleaseForRegistration() { + m_object = nullptr; + m_registeredRef = false; + if (m_refs == 0) + delete this; + } + }; + + struct KCMemoryRegion { + uint64_t vmAddress; + uint64_t size; + std::string prettyName; + }; + + struct BackingCacheMapping { + uint64_t vmAddress; + uint64_t size; + uint64_t fileOffset; + }; + + struct BackingCache { + std::string path; + bool isPrimary; + std::vector<BackingCacheMapping> mappings; + }; + + struct KCImageMemoryMapping { + std::string name; + uint64_t vmAddress; + uint64_t size; + bool loaded; + uint64_t rawViewOffset; + }; + + struct KCImage { + std::string name; + uint64_t headerFileAddress; + std::vector<KCImageMemoryMapping> mappings; + }; + + struct KCSymbol { + uint64_t address; + std::string name; + std::string image; + }; + + using namespace BinaryNinja; + struct KernelCacheMachOHeader : public KernelCacheCore::MetadataSerializable<KernelCacheMachOHeader> { + uint64_t textBase = 0; + uint64_t loadCommandOffset = 0; + mach_header_64 ident; + std::string identifierPrefix; + std::string installName; + + std::vector<std::pair<uint64_t, bool>> entryPoints; + std::vector<uint64_t> m_entryPoints; //list of entrypoints + + symtab_command symtab; + dysymtab_command dysymtab; + dyld_info_command dyldInfo; + routines_command_64 routines64; + function_starts_command functionStarts; + std::vector<section_64> moduleInitSections; + linkedit_data_command exportTrie; + linkedit_data_command chainedFixups {}; + + uint64_t relocationBase; + // Section and program headers, internally use 64-bit form as it is a superset of 32-bit + std::vector<segment_command_64> segments; //only three types of sections __TEXT, __DATA, __IMPORT + segment_command_64 linkeditSegment; + std::vector<section_64> sections; + std::vector<std::string> sectionNames; + + std::vector<section_64> symbolStubSections; + std::vector<section_64> symbolPointerSections; + + std::vector<std::string> dylibs; + + build_version_command buildVersion; + std::vector<build_tool_version> buildToolVersions; + + bool dysymPresent = false; + bool dyldInfoPresent = false; + bool exportTriePresent = false; + bool chainedFixupsPresent = false; + bool routinesPresent = false; + bool functionStartsPresent = false; + bool relocatable = false; + + void Store(KernelCacheCore::SerializationContext& context) const { + MSS(textBase); + MSS(loadCommandOffset); + MSS_SUBCLASS(ident); + MSS(identifierPrefix); + MSS(installName); + MSS(entryPoints); + MSS(m_entryPoints); + MSS_SUBCLASS(symtab); + MSS_SUBCLASS(dysymtab); + MSS_SUBCLASS(dyldInfo); + MSS_SUBCLASS(routines64); + MSS_SUBCLASS(functionStarts); + MSS_SUBCLASS(moduleInitSections); + MSS_SUBCLASS(exportTrie); + MSS_SUBCLASS(chainedFixups); + MSS(relocationBase); + MSS_SUBCLASS(segments); + MSS_SUBCLASS(linkeditSegment); + MSS_SUBCLASS(sections); + MSS(sectionNames); + MSS_SUBCLASS(symbolStubSections); + MSS_SUBCLASS(symbolPointerSections); + MSS(dylibs); + MSS_SUBCLASS(buildVersion); + MSS_SUBCLASS(buildToolVersions); + MSS(dysymPresent); + MSS(dyldInfoPresent); + MSS(exportTriePresent); + MSS(chainedFixupsPresent); + MSS(routinesPresent); + MSS(functionStartsPresent); + MSS(relocatable); + } + + static KernelCacheMachOHeader Load(KernelCacheCore::DeserializationContext& context) { + KernelCacheMachOHeader header; + header.MSL(textBase); + header.MSL(loadCommandOffset); + header.MSL(ident); + header.MSL(identifierPrefix); + header.MSL(installName); + header.MSL(entryPoints); + header.MSL(m_entryPoints); + header.MSL(symtab); + header.MSL(dysymtab); + header.MSL(dyldInfo); + header.MSL(routines64); + header.MSL(functionStarts); + header.MSL(moduleInitSections); + header.MSL(exportTrie); + header.MSL(chainedFixups); + header.MSL(relocationBase); + header.MSL(segments); + header.MSL(linkeditSegment); + header.MSL(sections); + header.MSL(sectionNames); + header.MSL(symbolStubSections); + header.MSL(symbolPointerSections); + header.MSL(dylibs); + header.MSL(buildVersion); + header.MSL(buildToolVersions); + header.MSL(dysymPresent); + header.MSL(dyldInfoPresent); + header.MSL(exportTriePresent); + header.MSL(chainedFixupsPresent); + header.MSL(routinesPresent); + header.MSL(functionStartsPresent); + header.MSL(relocatable); + return header; + } + }; + + + class KernelCache : public KCCoreRefCountObject<BNKernelCache, BNNewKernelCacheReference, BNFreeKernelCacheReference> { + public: + KernelCache(Ref<BinaryView> view); + + BNKCViewState GetState(); + static BNKCViewLoadProgress GetLoadProgress(Ref<BinaryView> view); + static uint64_t FastGetImageCount(Ref<BinaryView> view); + + bool LoadImageWithInstallName(std::string installName); + bool LoadImageContainingAddress(uint64_t addr); + std::vector<std::string> GetAvailableImages(); + + std::vector<KCSymbol> LoadAllSymbolsAndWait(); + + std::string GetNameForAddress(uint64_t address); + std::string GetImageNameForAddress(uint64_t address); + + std::vector<KCImage> GetImages(); + std::vector<KCImage> GetLoadedImages(); + + std::optional<KernelCacheMachOHeader> GetMachOHeaderForImage(std::string name); + std::optional<KernelCacheMachOHeader> GetMachOHeaderForAddress(uint64_t address); + }; +}
\ No newline at end of file diff --git a/view/kernelcache/api/kernelcachecore.h b/view/kernelcache/api/kernelcachecore.h new file mode 100644 index 00000000..d425bf15 --- /dev/null +++ b/view/kernelcache/api/kernelcachecore.h @@ -0,0 +1,133 @@ +#pragma once + + +#ifdef __cplusplus +extern "C" +{ +#endif + +#ifdef __GNUC__ + #ifdef KERNELCACHE_LIBRARY + #define KERNELCACHE_FFI_API __attribute__((visibility("default"))) + #else // KERNELCACHE_LIBRARY + #define KERNELCACHE_FFI_API + #endif // KERNELCACHE_LIBRARY +#else // __GNUC__ + #ifdef _MSC_VER + #ifndef DEMO_VERSION + #ifdef KERNELCACHE_LIBRARY + #define KERNELCACHE_FFI_API __declspec(dllexport) + #else // KERNELCACHE_LIBRARY + #define KERNELCACHE_FFI_API __declspec(dllimport) + #endif // KERNELCACHE_LIBRARY + #else + #define KERNELCACHE_FFI_API + #endif + #else // _MSC_VER + #define KERNELCACHE_FFI_API + #endif // _MSC_VER +#endif // __GNUC__C + +#define CORE_ALLOCATED_STRUCT(T) + +#define CORE_ALLOCATED_CLASS(T) \ + public: \ + CORE_ALLOCATED_STRUCT(T) \ + private: + +#define DECLARE_KERNELCACHE_API_OBJECT_INTERNAL(handle, cls, ns) \ + namespace ns { class cls; } struct handle { ns::cls* object; } + +#define DECLARE_KERNELCACHE_API_OBJECT(handle, cls) DECLARE_KERNELCACHE_API_OBJECT_INTERNAL(handle, cls, KernelCacheCore) + +#define IMPLEMENT_KERNELCACHE_API_OBJECT(handle) \ + CORE_ALLOCATED_CLASS(handle) \ + private: \ + handle m_apiObject; \ + public: \ + typedef handle* APIHandle; \ + handle* GetAPIObject() { return &m_apiObject; } \ + private: +#define INIT_KERNELCACHE_API_OBJECT() \ + m_apiObject.object = this; + + typedef enum BNKCViewState { + Unloaded, + Loaded, + LoadedWithImages, + } BNKCViewState; + + typedef enum BNKCViewLoadProgress { + LoadProgressNotStarted, + LoadProgressLoadingCaches, + LoadProgressLoadingImages, + LoadProgressFinished, + } BNKCViewLoadProgress; + + typedef struct BNBinaryView BNBinaryView; + typedef struct BNKernelCache BNKernelCache; + + typedef struct BNKCImageMemoryMapping { + char* name; + uint64_t vmAddress; + uint64_t size; + bool loaded; + uint64_t rawViewOffset; + } BNKCImageMemoryMapping; + + typedef struct BNKCImage { + char* name; + uint64_t headerFileAddress; + BNKCImageMemoryMapping* mappings; + size_t mappingCount; + } BNKCImage; + + typedef struct BNKCMappedMemoryRegion { + uint64_t vmAddress; + uint64_t size; + char* name; + } BNKCMappedMemoryRegion; + + typedef struct BNKCMemoryUsageInfo { + uint64_t sharedCacheRefs; + uint64_t mmapRefs; + } BNKCMemoryUsageInfo; + + typedef struct BNKCSymbolRep { + uint64_t address; + char* name; + char* image; + } BNKCSymbolRep; + + KERNELCACHE_FFI_API BNKernelCache* BNGetKernelCache(BNBinaryView* data); + + KERNELCACHE_FFI_API BNKernelCache* BNNewKernelCacheReference(BNKernelCache* cache); + KERNELCACHE_FFI_API void BNFreeKernelCacheReference(BNKernelCache* cache); + + KERNELCACHE_FFI_API char** BNKCViewGetInstallNames(BNKernelCache* cache, size_t* count); + + KERNELCACHE_FFI_API bool BNKCViewLoadImageWithInstallName(BNKernelCache* cache, char* name); + KERNELCACHE_FFI_API bool BNKCViewLoadImageContainingAddress(BNKernelCache* cache, uint64_t address); + + KERNELCACHE_FFI_API char* BNKCViewGetNameForAddress(BNKernelCache* cache, uint64_t address); + KERNELCACHE_FFI_API char* BNKCViewGetImageNameForAddress(BNKernelCache* cache, uint64_t address); + + KERNELCACHE_FFI_API BNKCViewState BNKCViewGetState(BNKernelCache* cache); + KERNELCACHE_FFI_API BNKCViewLoadProgress BNKCViewGetLoadProgress(uint64_t sessionID); + KERNELCACHE_FFI_API uint64_t BNKCViewFastGetImageCount(BNBinaryView* view); + + KERNELCACHE_FFI_API BNKCSymbolRep* BNKCViewLoadAllSymbolsAndWait(BNKernelCache* cache, size_t* count); + KERNELCACHE_FFI_API void BNKCViewFreeSymbols(BNKCSymbolRep* symbols, size_t count); + + KERNELCACHE_FFI_API BNKCImage* BNKCViewGetAllImages(BNKernelCache* cache, size_t* count); + KERNELCACHE_FFI_API void BNKCViewFreeAllImages(BNKCImage* images, size_t count); + + KERNELCACHE_FFI_API BNKCImage* BNKCViewGetLoadedImages(BNKernelCache* cache, size_t* count); + KERNELCACHE_FFI_API void BNKCViewFreeLoadedImages(BNKCImage* images, size_t count); + + KERNELCACHE_FFI_API char* BNKCViewGetImageHeaderForAddress(BNKernelCache* cache, uint64_t address); + KERNELCACHE_FFI_API char* BNKCViewGetImageHeaderForName(BNKernelCache* cache, char* name); + +#ifdef __cplusplus +} +#endif diff --git a/view/kernelcache/api/python/CMakeLists.txt b/view/kernelcache/api/python/CMakeLists.txt new file mode 100644 index 00000000..60fceeeb --- /dev/null +++ b/view/kernelcache/api/python/CMakeLists.txt @@ -0,0 +1,50 @@ +cmake_minimum_required(VERSION 3.9 FATAL_ERROR) + +project(kernelcache-python-api) + +file(GLOB PYTHON_SOURCES ${PROJECT_SOURCE_DIR}/*.py) +list(REMOVE_ITEM PYTHON_SOURCES ${PROJECT_SOURCE_DIR}/_kernelcachecore.py) +list(REMOVE_ITEM PYTHON_SOURCES ${PROJECT_SOURCE_DIR}/enums.py) + +add_executable(kernelcache_generator + ${PROJECT_SOURCE_DIR}/generator.cpp) +target_link_libraries(kernelcache_generator binaryninjaapi) +target_include_directories(kernelcache_generator PUBLIC {PROJECT_SOURCE_DIR}/../../api) + +set_target_properties(kernelcache_generator PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + BUILD_WITH_INSTALL_RPATH OFF + RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}) + +if(BN_INTERNAL_BUILD) + set(PYTHON_OUTPUT_DIRECTORY ${BN_RESOURCE_DIR}/python/binaryninja/kernelcache/) +else() + set(PYTHON_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/out/plugins/kernelcache/) +endif() + +if(WIN32) + if (BN_INTERNAL_BUILD) + add_custom_command(TARGET kernelcache_generator PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${BN_CORE_OUTPUT_DIR}/binaryninjacore.dll ${PROJECT_BINARY_DIR}/) + else() + add_custom_command(TARGET kernelcache_generator PRE_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${BN_INSTALL_DIR}/binaryninjacore.dll ${PROJECT_BINARY_DIR}/) + endif() +endif() + +add_custom_target(kernelcache_generator_copy ALL + BYPRODUCTS ${PROJECT_SOURCE_DIR}/_kernelcachecore.py ${PROJECT_SOURCE_DIR}/enums.py + DEPENDS ${PYTHON_SOURCES} ${PROJECT_SOURCE_DIR}/../kernelcachecore.h $<TARGET_FILE:kernelcache_generator> + COMMAND ${CMAKE_COMMAND} -E echo "Copying Shared Cache Python Sources" + COMMAND ${CMAKE_COMMAND} -E make_directory ${PYTHON_OUTPUT_DIRECTORY} + COMMAND ${CMAKE_COMMAND} -E env ASAN_OPTIONS=detect_leaks=0 $<TARGET_FILE:kernelcache_generator> + ${PROJECT_SOURCE_DIR}/../kernelcachecore.h + ${PROJECT_SOURCE_DIR}/_kernelcachecore.py + ${PROJECT_SOURCE_DIR}/_kernelcachecore_template.py + ${PROJECT_SOURCE_DIR}/kernelcache_enums.py + + COMMAND ${CMAKE_COMMAND} -E copy ${PYTHON_SOURCES} ${PYTHON_OUTPUT_DIRECTORY} + COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/_kernelcachecore.py ${PYTHON_OUTPUT_DIRECTORY} + COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_SOURCE_DIR}/kernelcache_enums.py ${PYTHON_OUTPUT_DIRECTORY}) + diff --git a/view/kernelcache/api/python/__init__.py b/view/kernelcache/api/python/__init__.py new file mode 100644 index 00000000..c6f05346 --- /dev/null +++ b/view/kernelcache/api/python/__init__.py @@ -0,0 +1,8 @@ +import os + +from binaryninja._binaryninjacore import BNGetUserPluginDirectory +user_plugin_dir = os.path.realpath(BNGetUserPluginDirectory()) +current_path = os.path.realpath(__file__) + +from .kernelcache import * + diff --git a/view/kernelcache/api/python/_kernelcachecore.py b/view/kernelcache/api/python/_kernelcachecore.py new file mode 100644 index 00000000..6bef0d06 --- /dev/null +++ b/view/kernelcache/api/python/_kernelcachecore.py @@ -0,0 +1,515 @@ +import binaryninja +import ctypes, os + +from typing import Optional +from . import kernelcache_enums +# Load core module +import platform +core = None +core_platform = platform.system() + +# By the time the debugger is loaded, binaryninja has not fully initialized. +# So we cannot call binaryninja.bundled_plugin_path() +from binaryninja._binaryninjacore import BNGetBundledPluginDirectory, BNFreeString +if core_platform == "Darwin": + _base_path = BNGetBundledPluginDirectory() + core = ctypes.CDLL(os.path.join(_base_path, "libkernelcache.dylib")) + +elif core_platform == "Linux": + _base_path = BNGetBundledPluginDirectory() + core = ctypes.CDLL(os.path.join(_base_path, "libkernelcache.so")) + +elif (core_platform == "Windows") or (core_platform.find("CYGWIN_NT") == 0): + _base_path = BNGetBundledPluginDirectory() + core = ctypes.CDLL(os.path.join(_base_path, "kernelcache.dll")) +else: + raise Exception("OS not supported") + +def cstr(var) -> Optional[ctypes.c_char_p]: + if var is None: + return None + if isinstance(var, bytes): + return var + return var.encode("utf-8") + +def pyNativeStr(arg): + if isinstance(arg, str): + return arg + else: + return arg.decode('utf8') + +def free_string(value:ctypes.c_char_p) -> None: + BNFreeString(ctypes.cast(value, ctypes.POINTER(ctypes.c_byte))) + +# Type definitions +from binaryninja._binaryninjacore import BNBinaryView, BNBinaryViewHandle +class BNKCImage(ctypes.Structure): + @property + def name(self): + return pyNativeStr(self._name) + @name.setter + def name(self, value): + self._name = cstr(value) +BNKCImageHandle = ctypes.POINTER(BNKCImage) +class BNKCImageMemoryMapping(ctypes.Structure): + @property + def name(self): + return pyNativeStr(self._name) + @name.setter + def name(self, value): + self._name = cstr(value) +BNKCImageMemoryMappingHandle = ctypes.POINTER(BNKCImageMemoryMapping) +class BNKCMappedMemoryRegion(ctypes.Structure): + @property + def name(self): + return pyNativeStr(self._name) + @name.setter + def name(self, value): + self._name = cstr(value) +BNKCMappedMemoryRegionHandle = ctypes.POINTER(BNKCMappedMemoryRegion) +class BNKCMemoryUsageInfo(ctypes.Structure): + pass +BNKCMemoryUsageInfoHandle = ctypes.POINTER(BNKCMemoryUsageInfo) +class BNKCSymbolRep(ctypes.Structure): + @property + def name(self): + return pyNativeStr(self._name) + @name.setter + def name(self, value): + self._name = cstr(value) + @property + def image(self): + return pyNativeStr(self._image) + @image.setter + def image(self, value): + self._image = cstr(value) +BNKCSymbolRepHandle = ctypes.POINTER(BNKCSymbolRep) +KCViewLoadProgressEnum = ctypes.c_int +KCViewStateEnum = ctypes.c_int +class BNKernelCache(ctypes.Structure): + pass +BNKernelCacheHandle = ctypes.POINTER(BNKernelCache) + +# Structure definitions +BNKCImage._fields_ = [ + ("_name", ctypes.c_char_p), + ("headerFileAddress", ctypes.c_ulonglong), + ("mappings", ctypes.POINTER(BNKCImageMemoryMapping)), + ("mappingCount", ctypes.c_ulonglong), + ] +BNKCImageMemoryMapping._fields_ = [ + ("_name", ctypes.c_char_p), + ("vmAddress", ctypes.c_ulonglong), + ("size", ctypes.c_ulonglong), + ("loaded", ctypes.c_bool), + ("rawViewOffset", ctypes.c_ulonglong), + ] +BNKCMappedMemoryRegion._fields_ = [ + ("vmAddress", ctypes.c_ulonglong), + ("size", ctypes.c_ulonglong), + ("_name", ctypes.c_char_p), + ] +BNKCMemoryUsageInfo._fields_ = [ + ("sharedCacheRefs", ctypes.c_ulonglong), + ("mmapRefs", ctypes.c_ulonglong), + ] +BNKCSymbolRep._fields_ = [ + ("address", ctypes.c_ulonglong), + ("_name", ctypes.c_char_p), + ("_image", ctypes.c_char_p), + ] + +# Function definitions +# ------------------------------------------------------- +# _BNFreeKernelCacheReference + +_BNFreeKernelCacheReference = core.BNFreeKernelCacheReference +_BNFreeKernelCacheReference.restype = None +_BNFreeKernelCacheReference.argtypes = [ + ctypes.POINTER(BNKernelCache), + ] + + +# noinspection PyPep8Naming +def BNFreeKernelCacheReference( + cache: ctypes.POINTER(BNKernelCache) + ) -> None: + return _BNFreeKernelCacheReference(cache) + + +# ------------------------------------------------------- +# _BNGetKernelCache + +_BNGetKernelCache = core.BNGetKernelCache +_BNGetKernelCache.restype = ctypes.POINTER(BNKernelCache) +_BNGetKernelCache.argtypes = [ + ctypes.POINTER(BNBinaryView), + ] + + +# noinspection PyPep8Naming +def BNGetKernelCache( + data: ctypes.POINTER(BNBinaryView) + ) -> Optional[ctypes.POINTER(BNKernelCache)]: + result = _BNGetKernelCache(data) + if not result: + return None + return result + + +# ------------------------------------------------------- +# _BNKCViewFastGetImageCount + +_BNKCViewFastGetImageCount = core.BNKCViewFastGetImageCount +_BNKCViewFastGetImageCount.restype = ctypes.c_ulonglong +_BNKCViewFastGetImageCount.argtypes = [ + ctypes.POINTER(BNBinaryView), + ] + + +# noinspection PyPep8Naming +def BNKCViewFastGetImageCount( + view: ctypes.POINTER(BNBinaryView) + ) -> int: + return _BNKCViewFastGetImageCount(view) + + +# ------------------------------------------------------- +# _BNKCViewFreeAllImages + +_BNKCViewFreeAllImages = core.BNKCViewFreeAllImages +_BNKCViewFreeAllImages.restype = None +_BNKCViewFreeAllImages.argtypes = [ + ctypes.POINTER(BNKCImage), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewFreeAllImages( + images: ctypes.POINTER(BNKCImage), + count: int + ) -> None: + return _BNKCViewFreeAllImages(images, count) + + +# ------------------------------------------------------- +# _BNKCViewFreeLoadedImages + +_BNKCViewFreeLoadedImages = core.BNKCViewFreeLoadedImages +_BNKCViewFreeLoadedImages.restype = None +_BNKCViewFreeLoadedImages.argtypes = [ + ctypes.POINTER(BNKCImage), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewFreeLoadedImages( + images: ctypes.POINTER(BNKCImage), + count: int + ) -> None: + return _BNKCViewFreeLoadedImages(images, count) + + +# ------------------------------------------------------- +# _BNKCViewFreeSymbols + +_BNKCViewFreeSymbols = core.BNKCViewFreeSymbols +_BNKCViewFreeSymbols.restype = None +_BNKCViewFreeSymbols.argtypes = [ + ctypes.POINTER(BNKCSymbolRep), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewFreeSymbols( + symbols: ctypes.POINTER(BNKCSymbolRep), + count: int + ) -> None: + return _BNKCViewFreeSymbols(symbols, count) + + +# ------------------------------------------------------- +# _BNKCViewGetAllImages + +_BNKCViewGetAllImages = core.BNKCViewGetAllImages +_BNKCViewGetAllImages.restype = ctypes.POINTER(BNKCImage) +_BNKCViewGetAllImages.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.POINTER(ctypes.c_ulonglong), + ] + + +# noinspection PyPep8Naming +def BNKCViewGetAllImages( + cache: ctypes.POINTER(BNKernelCache), + count: ctypes.POINTER(ctypes.c_ulonglong) + ) -> Optional[ctypes.POINTER(BNKCImage)]: + result = _BNKCViewGetAllImages(cache, count) + if not result: + return None + return result + + +# ------------------------------------------------------- +# _BNKCViewGetImageHeaderForAddress + +_BNKCViewGetImageHeaderForAddress = core.BNKCViewGetImageHeaderForAddress +_BNKCViewGetImageHeaderForAddress.restype = ctypes.POINTER(ctypes.c_byte) +_BNKCViewGetImageHeaderForAddress.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewGetImageHeaderForAddress( + cache: ctypes.POINTER(BNKernelCache), + address: int + ) -> Optional[Optional[str]]: + result = _BNKCViewGetImageHeaderForAddress(cache, address) + if not result: + return None + string = str(pyNativeStr(ctypes.cast(result, ctypes.c_char_p).value)) + BNFreeString(result) + return string + + +# ------------------------------------------------------- +# _BNKCViewGetImageHeaderForName + +_BNKCViewGetImageHeaderForName = core.BNKCViewGetImageHeaderForName +_BNKCViewGetImageHeaderForName.restype = ctypes.POINTER(ctypes.c_byte) +_BNKCViewGetImageHeaderForName.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.c_char_p, + ] + + +# noinspection PyPep8Naming +def BNKCViewGetImageHeaderForName( + cache: ctypes.POINTER(BNKernelCache), + name: Optional[str] + ) -> Optional[Optional[str]]: + result = _BNKCViewGetImageHeaderForName(cache, cstr(name)) + if not result: + return None + string = str(pyNativeStr(ctypes.cast(result, ctypes.c_char_p).value)) + BNFreeString(result) + return string + + +# ------------------------------------------------------- +# _BNKCViewGetImageNameForAddress + +_BNKCViewGetImageNameForAddress = core.BNKCViewGetImageNameForAddress +_BNKCViewGetImageNameForAddress.restype = ctypes.POINTER(ctypes.c_byte) +_BNKCViewGetImageNameForAddress.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewGetImageNameForAddress( + cache: ctypes.POINTER(BNKernelCache), + address: int + ) -> Optional[Optional[str]]: + result = _BNKCViewGetImageNameForAddress(cache, address) + if not result: + return None + string = str(pyNativeStr(ctypes.cast(result, ctypes.c_char_p).value)) + BNFreeString(result) + return string + + +# ------------------------------------------------------- +# _BNKCViewGetInstallNames + +_BNKCViewGetInstallNames = core.BNKCViewGetInstallNames +_BNKCViewGetInstallNames.restype = ctypes.POINTER(ctypes.c_char_p) +_BNKCViewGetInstallNames.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.POINTER(ctypes.c_ulonglong), + ] + + +# noinspection PyPep8Naming +def BNKCViewGetInstallNames( + cache: ctypes.POINTER(BNKernelCache), + count: ctypes.POINTER(ctypes.c_ulonglong) + ) -> Optional[ctypes.POINTER(ctypes.c_char_p)]: + result = _BNKCViewGetInstallNames(cache, count) + if not result: + return None + return result + + +# ------------------------------------------------------- +# _BNKCViewGetLoadProgress + +_BNKCViewGetLoadProgress = core.BNKCViewGetLoadProgress +_BNKCViewGetLoadProgress.restype = KCViewLoadProgressEnum +_BNKCViewGetLoadProgress.argtypes = [ + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewGetLoadProgress( + sessionID: int + ) -> KCViewLoadProgressEnum: + return _BNKCViewGetLoadProgress(sessionID) + + +# ------------------------------------------------------- +# _BNKCViewGetLoadedImages + +_BNKCViewGetLoadedImages = core.BNKCViewGetLoadedImages +_BNKCViewGetLoadedImages.restype = ctypes.POINTER(BNKCImage) +_BNKCViewGetLoadedImages.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.POINTER(ctypes.c_ulonglong), + ] + + +# noinspection PyPep8Naming +def BNKCViewGetLoadedImages( + cache: ctypes.POINTER(BNKernelCache), + count: ctypes.POINTER(ctypes.c_ulonglong) + ) -> Optional[ctypes.POINTER(BNKCImage)]: + result = _BNKCViewGetLoadedImages(cache, count) + if not result: + return None + return result + + +# ------------------------------------------------------- +# _BNKCViewGetNameForAddress + +_BNKCViewGetNameForAddress = core.BNKCViewGetNameForAddress +_BNKCViewGetNameForAddress.restype = ctypes.POINTER(ctypes.c_byte) +_BNKCViewGetNameForAddress.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewGetNameForAddress( + cache: ctypes.POINTER(BNKernelCache), + address: int + ) -> Optional[Optional[str]]: + result = _BNKCViewGetNameForAddress(cache, address) + if not result: + return None + string = str(pyNativeStr(ctypes.cast(result, ctypes.c_char_p).value)) + BNFreeString(result) + return string + + +# ------------------------------------------------------- +# _BNKCViewGetState + +_BNKCViewGetState = core.BNKCViewGetState +_BNKCViewGetState.restype = KCViewStateEnum +_BNKCViewGetState.argtypes = [ + ctypes.POINTER(BNKernelCache), + ] + + +# noinspection PyPep8Naming +def BNKCViewGetState( + cache: ctypes.POINTER(BNKernelCache) + ) -> KCViewStateEnum: + return _BNKCViewGetState(cache) + + +# ------------------------------------------------------- +# _BNKCViewLoadAllSymbolsAndWait + +_BNKCViewLoadAllSymbolsAndWait = core.BNKCViewLoadAllSymbolsAndWait +_BNKCViewLoadAllSymbolsAndWait.restype = ctypes.POINTER(BNKCSymbolRep) +_BNKCViewLoadAllSymbolsAndWait.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.POINTER(ctypes.c_ulonglong), + ] + + +# noinspection PyPep8Naming +def BNKCViewLoadAllSymbolsAndWait( + cache: ctypes.POINTER(BNKernelCache), + count: ctypes.POINTER(ctypes.c_ulonglong) + ) -> Optional[ctypes.POINTER(BNKCSymbolRep)]: + result = _BNKCViewLoadAllSymbolsAndWait(cache, count) + if not result: + return None + return result + + +# ------------------------------------------------------- +# _BNKCViewLoadImageContainingAddress + +_BNKCViewLoadImageContainingAddress = core.BNKCViewLoadImageContainingAddress +_BNKCViewLoadImageContainingAddress.restype = ctypes.c_bool +_BNKCViewLoadImageContainingAddress.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.c_ulonglong, + ] + + +# noinspection PyPep8Naming +def BNKCViewLoadImageContainingAddress( + cache: ctypes.POINTER(BNKernelCache), + address: int + ) -> bool: + return _BNKCViewLoadImageContainingAddress(cache, address) + + +# ------------------------------------------------------- +# _BNKCViewLoadImageWithInstallName + +_BNKCViewLoadImageWithInstallName = core.BNKCViewLoadImageWithInstallName +_BNKCViewLoadImageWithInstallName.restype = ctypes.c_bool +_BNKCViewLoadImageWithInstallName.argtypes = [ + ctypes.POINTER(BNKernelCache), + ctypes.c_char_p, + ] + + +# noinspection PyPep8Naming +def BNKCViewLoadImageWithInstallName( + cache: ctypes.POINTER(BNKernelCache), + name: Optional[str] + ) -> bool: + return _BNKCViewLoadImageWithInstallName(cache, cstr(name)) + + +# ------------------------------------------------------- +# _BNNewKernelCacheReference + +_BNNewKernelCacheReference = core.BNNewKernelCacheReference +_BNNewKernelCacheReference.restype = ctypes.POINTER(BNKernelCache) +_BNNewKernelCacheReference.argtypes = [ + ctypes.POINTER(BNKernelCache), + ] + + +# noinspection PyPep8Naming +def BNNewKernelCacheReference( + cache: ctypes.POINTER(BNKernelCache) + ) -> Optional[ctypes.POINTER(BNKernelCache)]: + result = _BNNewKernelCacheReference(cache) + if not result: + return None + return result + + + +# Helper functions +def handle_of_type(value, handle_type): + if isinstance(value, ctypes.POINTER(handle_type)) or isinstance(value, ctypes.c_void_p): + return ctypes.cast(value, ctypes.POINTER(handle_type)) + raise ValueError('expected pointer to %s' % str(handle_type)) diff --git a/view/kernelcache/api/python/_kernelcachecore_template.py b/view/kernelcache/api/python/_kernelcachecore_template.py new file mode 100644 index 00000000..c1f05e72 --- /dev/null +++ b/view/kernelcache/api/python/_kernelcachecore_template.py @@ -0,0 +1,43 @@ +import binaryninja +import ctypes, os + +from typing import Optional +from . import kernelcache_enums +# Load core module +import platform +core = None +core_platform = platform.system() + +# By the time the debugger is loaded, binaryninja has not fully initialized. +# So we cannot call binaryninja.bundled_plugin_path() +from binaryninja._binaryninjacore import BNGetBundledPluginDirectory, BNFreeString +if core_platform == "Darwin": + _base_path = BNGetBundledPluginDirectory() + core = ctypes.CDLL(os.path.join(_base_path, "libkernelcache.dylib")) + +elif core_platform == "Linux": + _base_path = BNGetBundledPluginDirectory() + core = ctypes.CDLL(os.path.join(_base_path, "libkernelcache.so")) + +elif (core_platform == "Windows") or (core_platform.find("CYGWIN_NT") == 0): + _base_path = BNGetBundledPluginDirectory() + core = ctypes.CDLL(os.path.join(_base_path, "kernelcache.dll")) +else: + raise Exception("OS not supported") + +def cstr(var) -> Optional[ctypes.c_char_p]: + if var is None: + return None + if isinstance(var, bytes): + return var + return var.encode("utf-8") + +def pyNativeStr(arg): + if isinstance(arg, str): + return arg + else: + return arg.decode('utf8') + +def free_string(value:ctypes.c_char_p) -> None: + BNFreeString(ctypes.cast(value, ctypes.POINTER(ctypes.c_byte))) + diff --git a/view/kernelcache/api/python/generator.cpp b/view/kernelcache/api/python/generator.cpp new file mode 100644 index 00000000..91ff88c0 --- /dev/null +++ b/view/kernelcache/api/python/generator.cpp @@ -0,0 +1,617 @@ +/* +Copyright 2020-2024 Vector 35 Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + + +#include <stdio.h> +#include <inttypes.h> +#include "binaryninjaapi.h" + +using namespace BinaryNinja; +using namespace std; + + +map<string, string> g_pythonKeywordReplacements = { + {"False", "False_"}, + {"True", "True_"}, + {"None", "None_"}, + {"and", "and_"}, + {"as", "as_"}, + {"assert", "assert_"}, + {"async", "async_"}, + {"await", "await_"}, + {"break", "break_"}, + {"class", "class_"}, + {"continue", "continue_"}, + {"def", "def_"}, + {"del", "del_"}, + {"elif", "elif_"}, + {"else", "else_"}, + {"except", "except_"}, + {"finally", "finally_"}, + {"for", "for_"}, + {"from", "from_"}, + {"global", "global_"}, + {"if", "if_"}, + {"import", "import_"}, + {"in", "in_"}, + {"is", "is_"}, + {"lambda", "lambda_"}, + {"nonlocal", "nonlocal_"}, + {"not", "not_"}, + {"or", "or_"}, + {"pass", "pass_"}, + {"raise", "raise_"}, + {"return", "return_"}, + {"try", "try_"}, + {"while", "while_"}, + {"with", "with_"}, + {"yield", "yield_"}, +}; + + +void OutputType(FILE* out, Type* type, bool isReturnType = false, bool isCallback = false) +{ + switch (type->GetClass()) + { + case BoolTypeClass: + fprintf(out, "ctypes.c_bool"); + break; + case IntegerTypeClass: + switch (type->GetWidth()) + { + case 1: + if (type->IsSigned()) + fprintf(out, "ctypes.c_byte"); + else + fprintf(out, "ctypes.c_ubyte"); + break; + case 2: + if (type->IsSigned()) + fprintf(out, "ctypes.c_short"); + else + fprintf(out, "ctypes.c_ushort"); + break; + case 4: + if (type->IsSigned()) + fprintf(out, "ctypes.c_int"); + else + fprintf(out, "ctypes.c_uint"); + break; + default: + if (type->IsSigned()) + fprintf(out, "ctypes.c_longlong"); + else + fprintf(out, "ctypes.c_ulonglong"); + break; + } + break; + case FloatTypeClass: + if (type->GetWidth() == 4) + fprintf(out, "ctypes.c_float"); + else + fprintf(out, "ctypes.c_double"); + break; + case NamedTypeReferenceClass: + if (type->GetNamedTypeReference()->GetTypeReferenceClass() == EnumNamedTypeClass) + { + string name = type->GetNamedTypeReference()->GetName().GetString(); + if (name.size() > 16 && name.substr(0, 11) == "_BNDebugger") + name = name.substr(3); + else if (name.size() > 15 && name.substr(0, 10) == "BNDebugger") + name = name.substr(2); + else if (name.size() > 15 && name.substr(0, 7) == "BNDebug") + name = name.substr(2); + else if (name.size() > 2 && name.substr(0, 2) == "BN") + name = name.substr(2); + fprintf(out, "%sEnum", name.c_str()); + } + else + { + fprintf(out, "%s", type->GetNamedTypeReference()->GetName().GetString().c_str()); + } + break; + case PointerTypeClass: + if (isCallback || (type->GetChildType()->GetClass() == VoidTypeClass)) + { + fprintf(out, "ctypes.c_void_p"); + break; + } + else if ((type->GetChildType()->GetClass() == IntegerTypeClass) && + (type->GetChildType()->GetWidth() == 1) && (type->GetChildType()->IsSigned())) + { + if (isReturnType) + fprintf(out, "ctypes.POINTER(ctypes.c_byte)"); + else + fprintf(out, "ctypes.c_char_p"); + break; + } + else if (type->GetChildType()->GetClass() == FunctionTypeClass) + { + fprintf(out, "ctypes.CFUNCTYPE("); + OutputType(out, type->GetChildType()->GetChildType(), true, true); + for (auto& i : type->GetChildType()->GetParameters()) + { + fprintf(out, ", "); + OutputType(out, i.type); + } + fprintf(out, ")"); + break; + } + fprintf(out, "ctypes.POINTER("); + OutputType(out, type->GetChildType()); + fprintf(out, ")"); + break; + case ArrayTypeClass: + OutputType(out, type->GetChildType()); + fprintf(out, " * %" PRId64, type->GetElementCount()); + break; + default: + fprintf(out, "None"); + break; + } +} + + +void OutputSwizzledType(FILE* out, Type* type) +{ + switch (type->GetClass()) + { + case BoolTypeClass: + fprintf(out, "bool"); + break; + case IntegerTypeClass: + fprintf(out, "int"); + break; + case FloatTypeClass: + fprintf(out, "float"); + break; + case NamedTypeReferenceClass: + if (type->GetNamedTypeReference()->GetTypeReferenceClass() == EnumNamedTypeClass) + { + string name = type->GetNamedTypeReference()->GetName().GetString(); + if (name.size() > 16 && name.substr(0, 11) == "_BNDebugger") + name = name.substr(3); + else if (name.size() > 15 && name.substr(0, 10) == "BNDebugger") + name = name.substr(2); + else if (name.size() > 15 && name.substr(0, 7) == "BNDebug") + name = name.substr(2); + else if (name.size() > 2 && name.substr(0, 2) == "BN") + name = name.substr(2); + fprintf(out, "%sEnum", name.c_str()); + } + else + { + fprintf(out, "%s", type->GetNamedTypeReference()->GetName().GetString().c_str()); + } + break; + case PointerTypeClass: + if (type->GetChildType()->GetClass() == VoidTypeClass) + { + fprintf(out, "Optional[ctypes.c_void_p]"); + break; + } + else if ((type->GetChildType()->GetClass() == IntegerTypeClass) && + (type->GetChildType()->GetWidth() == 1) && (type->GetChildType()->IsSigned())) + { + fprintf(out, "Optional[str]"); + break; + } + else if (type->GetChildType()->GetClass() == FunctionTypeClass) + { + fprintf(out, "ctypes.CFUNCTYPE("); + OutputType(out, type->GetChildType()->GetChildType(), true, true); + for (auto& i : type->GetChildType()->GetParameters()) + { + fprintf(out, ", "); + OutputType(out, i.type); + } + fprintf(out, ")"); + break; + } + fprintf(out, "ctypes.POINTER("); + OutputType(out, type->GetChildType()); + fprintf(out, ")"); + break; + case ArrayTypeClass: + OutputType(out, type->GetChildType()); + fprintf(out, " * %" PRId64, type->GetElementCount()); + break; + default: + fprintf(out, "None"); + break; + } +} + + +int main(int argc, char* argv[]) +{ + if (argc < 5) + { + fprintf(stderr, "Usage: generator <header> <output> <output_template> <output_enum>\n"); + return 1; + } + + // Parse API header to get type and function information + map<QualifiedName, Ref<Type>> types, vars, funcs; + string errors; + auto arch = new CoreArchitecture(BNGetNativeTypeParserArchitecture()); + + // Enable ephemeral settings + Settings::Instance()->LoadSettingsFile(""); + Settings::Instance()->Set("analysis.types.parserName", "ClangTypeParser"); + bool ok = arch->GetStandalonePlatform()->ParseTypesFromSourceFile(argv[1], types, vars, funcs, errors); + + if (!ok) + { + fprintf(stderr, "Errors: %s\n", errors.c_str()); + return 1; + } + + FILE* out = fopen(argv[2], "w"); + FILE* out_template = fopen(argv[3], "r"); + FILE* enums = fopen(argv[4], "w"); + + fprintf(enums, "import enum\n"); + + // Copy the content of the template to the output file + int c; + while((c = fgetc(out_template)) != EOF) + fputc(c, out); + + // Create type objects + fprintf(out, "# Type definitions\n"); + for (auto& i : types) + { + string name; + if (i.first.size() != 1) + continue; + name = i.first[0]; + if (name == "BNBinaryView") + { + fprintf(out, "from binaryninja._binaryninjacore import BNBinaryView, BNBinaryViewHandle\n"); + continue; + } + if (i.second->GetClass() == StructureTypeClass) + { + fprintf(out, "class %s(ctypes.Structure):\n", name.c_str()); + + // python uses str's, C uses byte-arrays + bool stringField = false; + for (auto& arg : i.second->GetStructure()->GetMembers()) + { + if ((arg.type->GetClass() == PointerTypeClass) && + (arg.type->GetChildType()->GetWidth() == 1) && + (arg.type->GetChildType()->IsSigned())) + { + fprintf(out, "\t@property\n\tdef %s(self):\n\t\treturn pyNativeStr(self._%s)\n", arg.name.c_str(), arg.name.c_str()); + fprintf(out, "\t@%s.setter\n\tdef %s(self, value):\n\t\tself._%s = cstr(value)\n", arg.name.c_str(), arg.name.c_str(), arg.name.c_str()); + stringField = true; + } + } + + if (!stringField) + fprintf(out, "\tpass\n"); + + fprintf(out, "%sHandle = ctypes.POINTER(%s)\n", name.c_str(), name.c_str()); + } + else if (i.second->GetClass() == EnumerationTypeClass) + { + bool isBNAPIEnum = false; + if (name.size() > 16 && name.substr(0, 11) == "_BNDebugger") + name = name.substr(3); + else if (name.size() > 15 && name.substr(0, 10) == "BNDebugger") + name = name.substr(2); + else if (name.size() > 15 && name.substr(0, 7) == "BNDebug") + name = name.substr(2); + else if (name.size() > 2 && name.substr(0, 2) == "BN") + { + name = name.substr(2); + isBNAPIEnum = false; + } + else + continue; + + if (isBNAPIEnum) + { + fprintf(out, "from binaryninja._binaryninjacore import %sEnum\n", name.c_str()); + continue; + } + + fprintf(out, "%sEnum = ctypes.c_int\n", name.c_str()); + + fprintf(enums, "\n\nclass %s(enum.IntEnum):\n", name.c_str()); + for (auto& j : i.second->GetEnumeration()->GetMembers()) + { + fprintf(enums, "\t%s = %" PRId64 "\n", j.name.c_str(), j.value); + } + } + else if ((i.second->GetClass() == BoolTypeClass) || (i.second->GetClass() == IntegerTypeClass) || + (i.second->GetClass() == FloatTypeClass) || (i.second->GetClass() == ArrayTypeClass)) + { + fprintf(out, "%s = ", name.c_str()); + OutputType(out, i.second); + fprintf(out, "\n"); + } + } + + + fprintf(out, "\n# Structure definitions\n"); + set<QualifiedName> structsToProcess; + set<QualifiedName> finishedStructs; + for (auto& i : types) + structsToProcess.insert(i.first); + while (structsToProcess.size() != 0) + { + set<QualifiedName> currentStructList = structsToProcess; + structsToProcess.clear(); + bool processedSome = false; + for (auto& i : currentStructList) + { + string name; + if (i.size() != 1) + continue; + Ref<Type> type = types[i]; + name = i[0]; + if ((type->GetClass() == StructureTypeClass) && (type->GetStructure()->GetMembers().size() != 0)) + { + bool requiresDependency = false; + for (auto& j : type->GetStructure()->GetMembers()) + { + if ((j.type->GetClass() == NamedTypeReferenceClass) && + (types[j.type->GetNamedTypeReference()->GetName()]->GetClass() == StructureTypeClass) && + (finishedStructs.count(j.type->GetNamedTypeReference()->GetName()) == 0)) + { + // This structure needs another structure that isn't fully defined yet, need to wait + // for the dependencies to be defined + structsToProcess.insert(i); + requiresDependency = true; + break; + } + } + + if (requiresDependency) + continue; + fprintf(out, "%s._fields_ = [\n", name.c_str()); + for (auto& j : type->GetStructure()->GetMembers()) + { + // To help the python->C wrappers + if ((j.type->GetClass() == PointerTypeClass) && + (j.type->GetChildType()->GetWidth() == 1) && + (j.type->GetChildType()->IsSigned())) + { + fprintf(out, "\t\t(\"_%s\", ", j.name.c_str()); + } + else + fprintf(out, "\t\t(\"%s\", ", j.name.c_str()); + OutputType(out, j.type); + fprintf(out, "),\n"); + } + fprintf(out, "\t]\n"); + finishedStructs.insert(i); + processedSome = true; + } + else if (type->GetClass() == NamedTypeReferenceClass) + { + if (type->GetNamedTypeReference()->GetTypeReferenceClass() == StructNamedTypeClass) + { + fprintf(out, "%s = %s\n", name.c_str(), type->GetNamedTypeReference()->GetName().GetString().c_str()); + fprintf(out, "%sHandle = %sHandle\n", name.c_str(), type->GetNamedTypeReference()->GetName().GetString().c_str()); + } + else if (type->GetNamedTypeReference()->GetTypeReferenceClass() == EnumNamedTypeClass) + { + fprintf(out, "%s = ctypes.c_int\n", name.c_str()); + } + finishedStructs.insert(i); + processedSome = true; + } + } + + if (!processedSome && structsToProcess.size() != 0) + { + fprintf(stderr, "Detected dependency cycle in structures\n"); + for (auto& i : structsToProcess) + fprintf(stderr, "%s\n", i.GetString().c_str()); + return 1; + } + } + + fprintf(out, "\n# Function definitions\n"); + for (auto& i : funcs) + { + string name; + if (i.first.size() != 1) + continue; + name = i.first[0]; + + // Check for a string result, these will be automatically wrapped to free the string + // memory and return a Python string + bool stringResult = (i.second->GetChildType()->GetClass() == PointerTypeClass) && + (i.second->GetChildType()->GetChildType()->GetWidth() == 1) && + (i.second->GetChildType()->GetChildType()->IsSigned()); + // Pointer returns will be automatically wrapped to return None on null pointer + bool pointerResult = (i.second->GetChildType()->GetClass() == PointerTypeClass); + + // From python -> C python3 requires str -> str.encode('charmap') + bool swizzleArgs = true; + if (name == "BNFreeString") + swizzleArgs = false; + + bool callbackConvention = false; + if (name == "BNAllocString") + { + // Don't perform automatic wrapping of string allocation, and return a void + // pointer so that callback functions (which is the only valid use of BNDebuggerAllocString) + // can properly return the result + stringResult = false; + callbackConvention = true; + swizzleArgs = false; + } + + string funcName = string("_") + name; + + fprintf(out, "# -------------------------------------------------------\n"); + fprintf(out, "# %s\n\n", funcName.c_str()); + fprintf(out, "%s = core.%s\n", funcName.c_str(), name.c_str()); + fprintf(out, "%s.restype = ", funcName.c_str()); + OutputType(out, i.second->GetChildType(), true, callbackConvention); + fprintf(out, "\n"); + if (!i.second->HasVariableArguments()) + { + fprintf(out, "%s.argtypes = [\n", funcName.c_str()); + for (auto& j : i.second->GetParameters()) + { + fprintf(out, "\t\t"); + if (name == "BNFreeString") + { + // BNDebuggerFreeString expects a pointer to a string allocated by the core, so do not use + // a c_char_p here, as that would be allocated by the Python runtime. This can + // be enforced by outputting like a return value. + OutputType(out, j.type, true); + } + else + { + OutputType(out, j.type); + } + fprintf(out, ",\n"); + } + fprintf(out, "\t]"); + } + else + { + // As of writing this, only BNLog's have variable instruction lengths, but in an attempt not to break in the future: + if (funcName.compare(0, 6, "_BNLog") == 0) + { + if (funcName != "_BNLog") + { + fprintf(out, "def %s(*args):\n", name.c_str()); + fprintf(out, "\treturn %s(*[cstr(arg) for arg in args])\n\n", funcName.c_str()); + continue; + } + else + { + fprintf(out, "def %s(level, *args):\n", name.c_str()); + fprintf(out, "\treturn %s(level, *[cstr(arg) for arg in args])\n\n", funcName.c_str()); + continue; + } + } + } + fprintf(out, "\n\n\n# noinspection PyPep8Naming\n"); + fprintf(out, "def %s(", name.c_str()); + if (!i.second->HasVariableArguments()) + { + size_t argN = 0; + for (auto& arg: i.second->GetParameters()) + { + string argName = arg.name; + if (g_pythonKeywordReplacements.find(argName) != g_pythonKeywordReplacements.end()) + argName = g_pythonKeywordReplacements[argName]; + + if (argName.empty()) + argName = "arg" + to_string(argN); + + if (argN > 0) + fprintf(out, ", "); + fprintf(out, "\n\t\t"); + fprintf(out, "%s: ", argName.c_str()); + if (swizzleArgs) + OutputSwizzledType(out, arg.type); + else + OutputType(out, arg.type); + argN ++; + } + } + fprintf(out, "\n\t\t) -> "); + if (swizzleArgs) + { + if (stringResult || pointerResult) + fprintf(out, "Optional["); + OutputSwizzledType(out, i.second->GetChildType()); + if (stringResult || pointerResult) + fprintf(out, "]"); + } + else + { + OutputType(out, i.second->GetChildType()); + } + fprintf(out, ":\n"); + + string stringArgFuncCall = funcName + "("; + size_t argN = 0; + for (auto& arg : i.second->GetParameters()) + { + string argName = arg.name; + if (g_pythonKeywordReplacements.find(argName) != g_pythonKeywordReplacements.end()) + argName = g_pythonKeywordReplacements[argName]; + + if (argName.empty()) + argName = "arg" + to_string(argN); + + if (swizzleArgs && (arg.type->GetClass() == PointerTypeClass) && + (arg.type->GetChildType()->GetClass() == IntegerTypeClass) && + (arg.type->GetChildType()->GetWidth() == 1) && + (arg.type->GetChildType()->IsSigned())) + { + stringArgFuncCall += string("cstr(") + argName + "), "; + } + else + { + stringArgFuncCall += argName + ", "; + } + argN++; + } + if (argN > 0) + stringArgFuncCall = stringArgFuncCall.substr(0, stringArgFuncCall.size()-2); + stringArgFuncCall += ")"; + + if (stringResult) + { + // Emit wrapper to get Python string and free native memory + fprintf(out, "\tresult = "); + fprintf(out, "%s\n", stringArgFuncCall.c_str()); + fprintf(out, "\tif not result:\n"); + fprintf(out, "\t\treturn None\n"); + fprintf(out, "\tstring = str(pyNativeStr(ctypes.cast(result, ctypes.c_char_p).value))\n"); + fprintf(out, "\tBNFreeString(result)\n"); + fprintf(out, "\treturn string\n"); + } + else if (pointerResult) + { + // Emit wrapper to return None on null pointer + fprintf(out, "\tresult = "); + fprintf(out, "%s\n", stringArgFuncCall.c_str()); + fprintf(out, "\tif not result:\n"); + fprintf(out, "\t\treturn None\n"); + fprintf(out, "\treturn result\n"); + } + else + { + fprintf(out, "\treturn "); + fprintf(out, "%s\n", stringArgFuncCall.c_str()); + } + fprintf(out, "\n\n"); + } + + fprintf(out, "\n# Helper functions\n"); + fprintf(out, "def handle_of_type(value, handle_type):\n"); + fprintf(out, "\tif isinstance(value, ctypes.POINTER(handle_type)) or isinstance(value, ctypes.c_void_p):\n"); + fprintf(out, "\t\treturn ctypes.cast(value, ctypes.POINTER(handle_type))\n"); + fprintf(out, "\traise ValueError('expected pointer to %%s' %% str(handle_type))\n"); + + fclose(out); + fclose(enums); + return 0; +} diff --git a/view/kernelcache/api/python/kernelcache.py b/view/kernelcache/api/python/kernelcache.py new file mode 100644 index 00000000..2148f2a8 --- /dev/null +++ b/view/kernelcache/api/python/kernelcache.py @@ -0,0 +1,238 @@ +import os +import ctypes +import dataclasses +import traceback + +import binaryninja +from binaryninja._binaryninjacore import BNFreeStringList, BNAllocString, BNFreeString + +from . import _kernelcachecore as kccore +from .kernelcache_enums import * + + +@dataclasses.dataclass +class KCImageMemoryMapping: + name: str + vmAddress: int + rawViewOffset: int + size: int + + def __str__(self): + return repr(self) + + def __repr__(self): + return (f"<KCImageMemoryMapping '{self.name}' {self.vmAddress:x}+{self.size:x} " + f"raw<{self.rawViewOffset:x}>>") + + +@dataclasses.dataclass +class KCImage: + name: str + headerFileAddress: int + mappings: list[KCImageMemoryMapping] + + def __str__(self): + return repr(self) + + def __repr__(self): + return f"<KCImage {self.name} @ {self.headerFileAddress:x}>" + + +@dataclasses.dataclass +class KCSymbol: + name: str + image: str + address: int + + def __str__(self): + return repr(self) + + def __repr__(self): + return f"<KCSymbol {self.name} @ {self.address:x} ({self.image})>" + + +@dataclasses.dataclass +class KernelCacheMachOHeader: + header: str + + @classmethod + def LoadFromString(cls, s: str): + return cls(header=s) + + def __repr__(self): + return f"<KernelCacheMachOHeader {self.header!r}>" + + +class KernelCache: + def __init__(self, view): + # Create a KernelCache object from a BinaryView. + self.handle = kccore.BNGetKernelCache(view.handle) + + @staticmethod + def get_load_progress(view) -> int: + """ + Returns the current load progress. + Note: In the C++ FFI this function takes the BinaryView's file session ID. + """ + return KCViewLoadProgress(kccore.BNKCViewGetLoadProgress(view.file.session_id)) + + @staticmethod + def fast_get_image_count(view) -> int: + """ + Quickly returns the number of images in the kernel cache. + """ + return kccore.BNKCViewFastGetImageCount(view.handle) + + def load_image_with_install_name(self, install_name: str) -> bool: + """ + Load a kernel cache image by its install name. + """ + return kccore.BNKCViewLoadImageWithInstallName(self.handle, install_name) + + def load_image_containing_address(self, addr: int) -> bool: + """ + Load the image that contains the given address. + """ + return kccore.BNKCViewLoadImageContainingAddress(self.handle, addr) + + @property + def image_names(self) -> list[str]: + """ + Return a list of available kernel cache image install names. + """ + count = ctypes.c_ulonglong() + value = kccore.BNKCViewGetInstallNames(self.handle, count) + if value is None: + return [] + result = [] + for i in range(count.value): + result.append(value[i].decode('utf-8')) + BNFreeStringList(value, count) + return result + + @property + def images(self) -> list[KCImage]: + """ + Return all kernel cache images. + """ + count = ctypes.c_ulonglong() + value = kccore.BNKCViewGetAllImages(self.handle, count) + if value is None: + return [] + result = [] + for i in range(count.value): + mappings = [] + for j in range(value[i].mappingCount): + mapping = KCImageMemoryMapping( + name=value[i].mappings[j].name, + vmAddress=value[i].mappings[j].vmAddress, + rawViewOffset=value[i].mappings[j].rawViewOffset, + size=value[i].mappings[j].size, + ) + mappings.append(mapping) + result.append(KCImage( + name=value[i].name, + headerFileAddress=value[i].headerFileAddress, + mappings=mappings + )) + kccore.BNKCViewFreeAllImages(value, count) + return result + + @property + def loaded_images(self) -> list[KCImage]: + """ + Return the kernel cache images that are currently loaded. + """ + count = ctypes.c_ulonglong() + images = kccore.BNKCViewGetLoadedImages(self.handle, count) + if images is None: + return [] + result = [] + for i in range(count.value): + mappings = [] + for j in range(images[i].mappingCount): + mapping = KCImageMemoryMapping( + name=images[i].mappings[j].name, + vmAddress=images[i].mappings[j].vmAddress, + rawViewOffset=images[i].mappings[j].rawViewOffset, + size=images[i].mappings[j].size, + ) + mappings.append(mapping) + result.append(KCImage( + name=images[i].name, + headerFileAddress=images[i].headerFileAddress, + mappings=mappings + )) + kccore.BNKCViewFreeAllImages(images, count) + return result + + def load_all_symbols_and_wait(self) -> list[KCSymbol]: + """ + Load all symbols from the kernel cache and wait for completion. + """ + count = ctypes.c_ulonglong() + value = kccore.BNKCViewLoadAllSymbolsAndWait(self.handle, count) + if value is None: + return [] + result = [] + for i in range(count.value): + sym = KCSymbol( + name=value[i].name, + image=value[i].image, + address=value[i].address + ) + result.append(sym) + kccore.BNKCViewFreeSymbols(value, count) + return result + + def get_name_for_address(self, address: int) -> str: + """ + Return the symbol name for a given address. + """ + name = kccore.BNKCViewGetNameForAddress(self.handle, address) + if name is None: + return "" + return name + + def get_image_name_for_address(self, address: int) -> str: + """ + Return the image name for a given address. + """ + name = kccore.BNKCViewGetImageNameForAddress(self.handle, address) + if name is None: + return "" + return name + + def get_macho_header_for_image(self, name: str): + """ + Return a KernelCacheMachOHeader for the image with the given install name. + """ + s = BNAllocString(name) + outputStr = kccore.BNKCViewGetImageHeaderForName(self.handle, s) + if outputStr is None: + return None + output = outputStr + BNFreeString(outputStr) + if output == "": + return None + return KernelCacheMachOHeader.LoadFromString(output) + + def get_macho_header_for_address(self, address: int): + """ + Return a KernelCacheMachOHeader for the image containing the given address. + """ + outputStr = kccore.BNKCViewGetImageHeaderForAddress(self.handle, address) + if outputStr is None: + return None + output = outputStr + BNFreeString(outputStr) + if output == "": + return None + return KernelCacheMachOHeader.LoadFromString(output) + + @property + def state(self): + """ + Return the current state of the kernel cache. + """ + return KCViewState(kccore.BNKCViewGetState(self.handle)) diff --git a/view/kernelcache/api/python/kernelcache_enums.py b/view/kernelcache/api/python/kernelcache_enums.py new file mode 100644 index 00000000..e610c527 --- /dev/null +++ b/view/kernelcache/api/python/kernelcache_enums.py @@ -0,0 +1,14 @@ +import enum + + +class KCViewLoadProgress(enum.IntEnum): + LoadProgressNotStarted = 0 + LoadProgressLoadingCaches = 1 + LoadProgressLoadingImages = 2 + LoadProgressFinished = 3 + + +class KCViewState(enum.IntEnum): + Unloaded = 0 + Loaded = 1 + LoadedWithImages = 2 diff --git a/view/kernelcache/core/CMakeLists.txt b/view/kernelcache/core/CMakeLists.txt new file mode 100644 index 00000000..d2b37443 --- /dev/null +++ b/view/kernelcache/core/CMakeLists.txt @@ -0,0 +1,103 @@ +cmake_minimum_required(VERSION 3.13 FATAL_ERROR) + +project(kernelcachecore) + +if((NOT BN_API_PATH) AND (NOT BN_INTERNAL_BUILD)) + set(BN_API_PATH $ENV{BN_API_PATH}) + if(NOT BN_API_PATH) + message(FATAL_ERROR "Provide path to Binary Ninja API source in BN_API_PATH") + endif() +endif() +# recursive source glob +file (GLOB_RECURSE COMMON_SOURCES CONFIGURE_DEPENDS + *.h + *.hpp + *.c + *.cpp + ) + +set(SOURCES ${COMMON_SOURCES}) + +add_library(kernelcachecore OBJECT ${SOURCES}) + +function(get_recursive_include_dirs target result) + # Initialize an empty list to store include directories + set(include_dirs "") + + # Get the include directories of the current target + get_target_property(current_target_includes ${target} INTERFACE_INCLUDE_DIRECTORIES) + if(current_target_includes) + list(APPEND include_dirs ${current_target_includes}) + endif() + + # Get the libraries that this target links to + get_target_property(linked_libraries ${target} INTERFACE_LINK_LIBRARIES) + if(linked_libraries) + foreach(linked_library IN LISTS linked_libraries) + # Skip plain library names (non-target libraries) + if(TARGET ${linked_library}) + # Recursively get include directories from linked libraries + get_recursive_include_dirs(${linked_library} linked_library_includes) + list(APPEND include_dirs ${linked_library_includes}) + endif() + endforeach() + endif() + + # Set the result to the collected include directories + set(${result} ${include_dirs} PARENT_SCOPE) +endfunction() + +get_recursive_include_dirs(binaryninjaapi INCLUDES) + + +set(COMPILE_DEFS "") + +if (HARD_FAIL_MODE) + set(COMPILE_DEFS "${COMPILE_DEFS} ABORT_FAILURES;") +endif() + +if (BN_REF_COUNT_DEBUG) + set(COMPILE_DEFS "${COMPILE_DEFS} BN_REF_COUNT_DEBUG;") +endif() + +if (SLIDEINFO_DEBUG_TAGS) + set(COMPILE_DEFS "${COMPILE_DEFS} SLIDEINFO_DEBUG_TAGS;") +endif() + +if (KC_VIEW_NAME) + set(COMPILE_DEFS "${COMPILE_DEFS} KC_VIEW_NAME=\"${KC_VIEW_NAME}\";") +else() + message(FATAL_ERROR "No view name provided. Fatal.") +endif() + +if (METADATA_VERSION) + set(COMPILE_DEFS "${COMPILE_DEFS} METADATA_VERSION=${METADATA_VERSION};") +else() + message(FATAL_ERROR "No metadata version provided. Fatal.") +endif() + +target_compile_definitions(kernelcachecore PRIVATE ${COMPILE_DEFS}) + + +target_compile_definitions(kernelcachecore PRIVATE KERNELCACHE_LIBRARY ${COMPILE_DEFS}) + +target_include_directories(kernelcachecore PRIVATE ${CMAKE_CURRENT_SOURCE_DIR} ${INCLUDES} ${CMAKE_CURRENT_SOURCE_DIR}/transformers) + +set_target_properties(kernelcachecore PROPERTIES + CXX_STANDARD 17 + CXX_VISIBILITY_PRESET hidden + CXX_STANDARD_REQUIRED ON + VISIBILITY_INLINES_HIDDEN ON + POSITION_INDEPENDENT_CODE ON + ) + +if(BN_INTERNAL_BUILD) + set(LIBRARY_OUTPUT_DIRECTORY_PATH "${BN_CORE_PLUGIN_DIR}") +else() + set(LIBRARY_OUTPUT_DIRECTORY_PATH "${CMAKE_BINARY_DIR}/out/plugins") +endif() + +set_target_properties(kernelcachecore PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${LIBRARY_OUTPUT_DIRECTORY_PATH} + RUNTIME_OUTPUT_DIRECTORY ${LIBRARY_OUTPUT_DIRECTORY_PATH} + ) diff --git a/view/kernelcache/core/KCView.cpp b/view/kernelcache/core/KCView.cpp new file mode 100644 index 00000000..f166eafd --- /dev/null +++ b/view/kernelcache/core/KCView.cpp @@ -0,0 +1,921 @@ +// +// Created by kat on 5/23/23. +// + +/* + * + * */ + +#include "KCView.h" +#include "view/macho/machoview.h" +#include "KernelCache.h" + +[[maybe_unused]] KCViewType* g_kcViewType; + + +#define COMPRESSION_DEBUG 1 + +using namespace BinaryNinja; + + +KCView::KCView(const std::string& typeName, BinaryView* data, bool parseOnly) : + BinaryView(typeName, data->GetFile(), data), m_parseOnly(parseOnly) +{ + CreateLogger("KernelCache"); +} + +KCView::~KCView() +{ +} + +enum KCPlatform { + KCPlatformMacOS = 1, + KCPlatformiOS = 2, + KCPlatformTVOS = 3, + KCPlatformWatchOS = 4, + KCPlatformBridgeOS = 5, // T1/T2 APL1023/T8012, this is your touchbar/touchid in intel macs. Similar to watchOS. + // KCPlatformMacCatalyst = 6, + KCPlatformiOSSimulator = 7, + KCPlatformTVOSSimulator = 8, + KCPlatformWatchOSSimulator = 9, + KCPlatformVisionOS = 11, // Apple Vision Pro + KCPlatformVisionOSSimulator = 12 // Apple Vision Pro Simulator +}; + +bool KCView::Init() +{ + BinaryReader reader(GetParentView()); + reader.Seek(0x4); + uint32_t cpuType = reader.Read32(); + reader.Seek(0x10); + uint64_t ncmds = reader.Read32(); + uint64_t sizeofcmds = reader.Read32(); + + // get __TEXT seg offset + uint64_t textSegOffset = 0; + uint64_t anyFilesetHeaderFileOffset = 0; + uint64_t offset = 0x20; + for (uint64_t i = 0; i < ncmds; i++) + { + reader.Seek(offset); + uint64_t cmd = reader.Read32(); + uint64_t cmdsize = reader.Read32(); + if (textSegOffset == 0 && cmd == LC_SEGMENT_64) + { + uint64_t segname = reader.Read64(); + reader.Read64(); + uint64_t vmaddr = reader.Read64(); + if (segname == 0x545845545f5f) // __TEXT + { + textSegOffset = vmaddr; + } + } + + if (anyFilesetHeaderFileOffset == 0 && cmd == LC_FILESET_ENTRY) + { + reader.SeekRelative(8); + anyFilesetHeaderFileOffset = reader.Read64(); + } + + offset += cmdsize; + if (textSegOffset && anyFilesetHeaderFileOffset) + break; + } + + Ref<Platform> platform; + Ref<Architecture> architecture; + + if (anyFilesetHeaderFileOffset) + { + reader.Seek(anyFilesetHeaderFileOffset); + reader.SeekRelative(0x10); + + uint64_t ncmdsFH = reader.Read32(); + + reader.Seek(anyFilesetHeaderFileOffset + 0x20); + offset = anyFilesetHeaderFileOffset + 0x20; + bool foundBuildVersion = false; + for (uint64_t i = 0; i < ncmdsFH; i++) + { + reader.Seek(offset); + uint64_t cmd = reader.Read32(); + uint64_t cmdsize = reader.Read32(); + if (cmd == LC_BUILD_VERSION) + { + uint32_t platformID = reader.Read32(); + std::map<std::string, Ref<Metadata>> metadataMap = { + {"machoplatform", new Metadata((uint64_t) platformID)}, + }; + Ref<Metadata> metadata = new Metadata(metadataMap); + Ref<Platform> plat = g_kcViewType->RecognizePlatform(cpuType, GetDefaultEndianness(), this, metadata); + SetDefaultArchitecture(plat->GetArchitecture()); + SetDefaultPlatform(plat); + foundBuildVersion = true; + } + offset += cmdsize; + } + if (!foundBuildVersion) + { + LogError("Failed to find LC_BUILD_VERSION in subheader"); + SetDefaultArchitecture(Architecture::GetByName("aarch64")); + SetDefaultPlatform(Platform::GetByName("macos-kernel-aarch64")); + } + } + else + { + LogError("MH_FILESET had no subheaders"); + return false; + } + + if (textSegOffset == 0) + { + LogError("Failed to find __TEXT segment"); + return false; + } + + + // Add Mach-O file header type info + EnumerationBuilder cpuTypeBuilder; + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_ANY", MACHO_CPU_TYPE_ANY); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_VAX", MACHO_CPU_TYPE_VAX); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_MC680x0", MACHO_CPU_TYPE_MC680x0); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_X86", MACHO_CPU_TYPE_X86); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_X86_64", MACHO_CPU_TYPE_X86_64); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_MIPS", MACHO_CPU_TYPE_MIPS); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_MC98000", MACHO_CPU_TYPE_MC98000); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_HPPA", MACHO_CPU_TYPE_HPPA); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_ARM", MACHO_CPU_TYPE_ARM); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_ARM64", MACHO_CPU_TYPE_ARM64); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_ARM64_32", MACHO_CPU_TYPE_ARM64_32); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_MC88000", MACHO_CPU_TYPE_MC88000); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_SPARC", MACHO_CPU_TYPE_SPARC); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_I860", MACHO_CPU_TYPE_I860); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_ALPHA", MACHO_CPU_TYPE_ALPHA); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_POWERPC", MACHO_CPU_TYPE_POWERPC); + cpuTypeBuilder.AddMemberWithValue("CPU_TYPE_POWERPC64", MACHO_CPU_TYPE_POWERPC64); + Ref<Enumeration> cpuTypeEnum = cpuTypeBuilder.Finalize(); + + Ref<Type> cpuTypeEnumType = Type::EnumerationType(nullptr, cpuTypeEnum, 4, false); + std::string cpuTypeEnumName = "cpu_type_t"; + std::string cpuTypeEnumId = Type::GenerateAutoTypeId("macho", cpuTypeEnumName); + DefineType(cpuTypeEnumId, cpuTypeEnumName, cpuTypeEnumType); + + EnumerationBuilder fileTypeBuilder; + fileTypeBuilder.AddMemberWithValue("MH_OBJECT", MH_OBJECT); + fileTypeBuilder.AddMemberWithValue("MH_EXECUTE", MH_EXECUTE); + fileTypeBuilder.AddMemberWithValue("MH_FVMLIB", MH_FVMLIB); + fileTypeBuilder.AddMemberWithValue("MH_CORE", MH_CORE); + fileTypeBuilder.AddMemberWithValue("MH_PRELOAD", MH_PRELOAD); + fileTypeBuilder.AddMemberWithValue("MH_DYLIB", MH_DYLIB); + fileTypeBuilder.AddMemberWithValue("MH_DYLINKER", MH_DYLINKER); + fileTypeBuilder.AddMemberWithValue("MH_BUNDLE", MH_BUNDLE); + fileTypeBuilder.AddMemberWithValue("MH_DYLIB_STUB", MH_DYLIB_STUB); + fileTypeBuilder.AddMemberWithValue("MH_DSYM", MH_DSYM); + fileTypeBuilder.AddMemberWithValue("MH_KEXT_BUNDLE", MH_KEXT_BUNDLE); + fileTypeBuilder.AddMemberWithValue("MH_FILESET", MH_FILESET); + Ref<Enumeration> fileTypeEnum = fileTypeBuilder.Finalize(); + + Ref<Type> fileTypeEnumType = Type::EnumerationType(nullptr, fileTypeEnum, 4, false); + std::string fileTypeEnumName = "file_type_t"; + std::string fileTypeEnumId = Type::GenerateAutoTypeId("macho", fileTypeEnumName); + DefineType(fileTypeEnumId, fileTypeEnumName, fileTypeEnumType); + + EnumerationBuilder flagsTypeBuilder; + flagsTypeBuilder.AddMemberWithValue("MH_NOUNDEFS", MH_NOUNDEFS); + flagsTypeBuilder.AddMemberWithValue("MH_INCRLINK", MH_INCRLINK); + flagsTypeBuilder.AddMemberWithValue("MH_DYLDLINK", MH_DYLDLINK); + flagsTypeBuilder.AddMemberWithValue("MH_BINDATLOAD", MH_BINDATLOAD); + flagsTypeBuilder.AddMemberWithValue("MH_PREBOUND", MH_PREBOUND); + flagsTypeBuilder.AddMemberWithValue("MH_SPLIT_SEGS", MH_SPLIT_SEGS); + flagsTypeBuilder.AddMemberWithValue("MH_LAZY_INIT", MH_LAZY_INIT); + flagsTypeBuilder.AddMemberWithValue("MH_TWOLEVEL", MH_TWOLEVEL); + flagsTypeBuilder.AddMemberWithValue("MH_FORCE_FLAT", MH_FORCE_FLAT); + flagsTypeBuilder.AddMemberWithValue("MH_NOMULTIDEFS", MH_NOMULTIDEFS); + flagsTypeBuilder.AddMemberWithValue("MH_NOFIXPREBINDING", MH_NOFIXPREBINDING); + flagsTypeBuilder.AddMemberWithValue("MH_PREBINDABLE", MH_PREBINDABLE); + flagsTypeBuilder.AddMemberWithValue("MH_ALLMODSBOUND", MH_ALLMODSBOUND); + flagsTypeBuilder.AddMemberWithValue("MH_SUBSECTIONS_VIA_SYMBOLS", MH_SUBSECTIONS_VIA_SYMBOLS); + flagsTypeBuilder.AddMemberWithValue("MH_CANONICAL", MH_CANONICAL); + flagsTypeBuilder.AddMemberWithValue("MH_WEAK_DEFINES", MH_WEAK_DEFINES); + flagsTypeBuilder.AddMemberWithValue("MH_BINDS_TO_WEAK", MH_BINDS_TO_WEAK); + flagsTypeBuilder.AddMemberWithValue("MH_ALLOW_STACK_EXECUTION", MH_ALLOW_STACK_EXECUTION); + flagsTypeBuilder.AddMemberWithValue("MH_ROOT_SAFE", MH_ROOT_SAFE); + flagsTypeBuilder.AddMemberWithValue("MH_SETUID_SAFE", MH_SETUID_SAFE); + flagsTypeBuilder.AddMemberWithValue("MH_NO_REEXPORTED_DYLIBS", MH_NO_REEXPORTED_DYLIBS); + flagsTypeBuilder.AddMemberWithValue("MH_PIE", MH_PIE); + flagsTypeBuilder.AddMemberWithValue("MH_DEAD_STRIPPABLE_DYLIB", MH_DEAD_STRIPPABLE_DYLIB); + flagsTypeBuilder.AddMemberWithValue("MH_HAS_TLV_DESCRIPTORS", MH_HAS_TLV_DESCRIPTORS); + flagsTypeBuilder.AddMemberWithValue("MH_NO_HEAP_EXECUTION", MH_NO_HEAP_EXECUTION); + flagsTypeBuilder.AddMemberWithValue("MH_APP_EXTENSION_SAFE", _MH_APP_EXTENSION_SAFE); + flagsTypeBuilder.AddMemberWithValue("MH_NLIST_OUTOFSYNC_WITH_DYLDINFO", _MH_NLIST_OUTOFSYNC_WITH_DYLDINFO); + flagsTypeBuilder.AddMemberWithValue("MH_SIM_SUPPORT", _MH_SIM_SUPPORT); + flagsTypeBuilder.AddMemberWithValue("MH_DYLIB_IN_CACHE", _MH_DYLIB_IN_CACHE); + Ref<Enumeration> flagsTypeEnum = flagsTypeBuilder.Finalize(); + + Ref<Type> flagsTypeEnumType = Type::EnumerationType(nullptr, flagsTypeEnum, 4, false); + std::string flagsTypeEnumName = "flags_type_t"; + std::string flagsTypeEnumId = Type::GenerateAutoTypeId("macho", flagsTypeEnumName); + DefineType(flagsTypeEnumId, flagsTypeEnumName, flagsTypeEnumType); + + StructureBuilder machoHeaderBuilder; + machoHeaderBuilder.AddMember(Type::IntegerType(4, false), "magic"); + machoHeaderBuilder.AddMember(Type::NamedType(this, QualifiedName("cpu_type_t")), "cputype"); + machoHeaderBuilder.AddMember(Type::IntegerType(4, false), "cpusubtype"); + machoHeaderBuilder.AddMember(Type::NamedType(this, QualifiedName("file_type_t")), "filetype"); + machoHeaderBuilder.AddMember(Type::IntegerType(4, false), "ncmds"); + machoHeaderBuilder.AddMember(Type::IntegerType(4, false), "sizeofcmds"); + machoHeaderBuilder.AddMember(Type::NamedType(this, QualifiedName("flags_type_t")), "flags"); + if (GetAddressSize() == 8) + machoHeaderBuilder.AddMember(Type::IntegerType(4, false), "reserved"); + Ref<Structure> machoHeaderStruct = machoHeaderBuilder.Finalize(); + QualifiedName headerName = (GetAddressSize() == 8) ? std::string("mach_header_64") : std::string("mach_header"); + + std::string headerTypeId = Type::GenerateAutoTypeId("macho", headerName); + Ref<Type> machoHeaderType = Type::StructureType(machoHeaderStruct); + DefineType(headerTypeId, headerName, machoHeaderType); + + EnumerationBuilder cmdTypeBuilder; + cmdTypeBuilder.AddMemberWithValue("LC_REQ_DYLD", LC_REQ_DYLD); + cmdTypeBuilder.AddMemberWithValue("LC_SEGMENT", LC_SEGMENT); + cmdTypeBuilder.AddMemberWithValue("LC_SYMTAB", LC_SYMTAB); + cmdTypeBuilder.AddMemberWithValue("LC_SYMSEG", LC_SYMSEG); + cmdTypeBuilder.AddMemberWithValue("LC_THREAD", LC_THREAD); + cmdTypeBuilder.AddMemberWithValue("LC_UNIXTHREAD", LC_UNIXTHREAD); + cmdTypeBuilder.AddMemberWithValue("LC_LOADFVMLIB", LC_LOADFVMLIB); + cmdTypeBuilder.AddMemberWithValue("LC_IDFVMLIB", LC_IDFVMLIB); + cmdTypeBuilder.AddMemberWithValue("LC_IDENT", LC_IDENT); + cmdTypeBuilder.AddMemberWithValue("LC_FVMFILE", LC_FVMFILE); + cmdTypeBuilder.AddMemberWithValue("LC_PREPAGE", LC_PREPAGE); + cmdTypeBuilder.AddMemberWithValue("LC_DYSYMTAB", LC_DYSYMTAB); + cmdTypeBuilder.AddMemberWithValue("LC_LOAD_DYLIB", LC_LOAD_DYLIB); + cmdTypeBuilder.AddMemberWithValue("LC_ID_DYLIB", LC_ID_DYLIB); + cmdTypeBuilder.AddMemberWithValue("LC_LOAD_DYLINKER", LC_LOAD_DYLINKER); + cmdTypeBuilder.AddMemberWithValue("LC_ID_DYLINKER", LC_ID_DYLINKER); + cmdTypeBuilder.AddMemberWithValue("LC_PREBOUND_DYLIB", LC_PREBOUND_DYLIB); + cmdTypeBuilder.AddMemberWithValue("LC_ROUTINES", LC_ROUTINES); + cmdTypeBuilder.AddMemberWithValue("LC_SUB_FRAMEWORK", LC_SUB_FRAMEWORK); + cmdTypeBuilder.AddMemberWithValue("LC_SUB_UMBRELLA", LC_SUB_UMBRELLA); + cmdTypeBuilder.AddMemberWithValue("LC_SUB_CLIENT", LC_SUB_CLIENT); + cmdTypeBuilder.AddMemberWithValue("LC_SUB_LIBRARY", LC_SUB_LIBRARY); + cmdTypeBuilder.AddMemberWithValue("LC_TWOLEVEL_HINTS", LC_TWOLEVEL_HINTS); + cmdTypeBuilder.AddMemberWithValue("LC_PREBIND_CKSUM", LC_PREBIND_CKSUM); + cmdTypeBuilder.AddMemberWithValue("LC_LOAD_WEAK_DYLIB", LC_LOAD_WEAK_DYLIB); // (0x18 | LC_REQ_DYLD) + cmdTypeBuilder.AddMemberWithValue("LC_SEGMENT_64", LC_SEGMENT_64); + cmdTypeBuilder.AddMemberWithValue("LC_ROUTINES_64", LC_ROUTINES_64); + cmdTypeBuilder.AddMemberWithValue("LC_UUID", LC_UUID); + cmdTypeBuilder.AddMemberWithValue("LC_RPATH", LC_RPATH); // (0x1c | LC_REQ_DYLD) + cmdTypeBuilder.AddMemberWithValue("LC_CODE_SIGNATURE", LC_CODE_SIGNATURE); + cmdTypeBuilder.AddMemberWithValue("LC_SEGMENT_SPLIT_INFO", LC_SEGMENT_SPLIT_INFO); + cmdTypeBuilder.AddMemberWithValue("LC_REEXPORT_DYLIB", LC_REEXPORT_DYLIB); // (0x1f | LC_REQ_DYLD) + cmdTypeBuilder.AddMemberWithValue("LC_LAZY_LOAD_DYLIB", LC_LAZY_LOAD_DYLIB); + cmdTypeBuilder.AddMemberWithValue("LC_ENCRYPTION_INFO", LC_ENCRYPTION_INFO); + cmdTypeBuilder.AddMemberWithValue("LC_DYLD_INFO", LC_DYLD_INFO); + cmdTypeBuilder.AddMemberWithValue("LC_DYLD_INFO_ONLY", LC_DYLD_INFO_ONLY); // (0x22 | LC_REQ_DYLD) + cmdTypeBuilder.AddMemberWithValue("LC_LOAD_UPWARD_DYLIB", LC_LOAD_UPWARD_DYLIB); // (0x23 | LC_REQ_DYLD) + cmdTypeBuilder.AddMemberWithValue("LC_VERSION_MIN_MACOSX", LC_VERSION_MIN_MACOSX); + cmdTypeBuilder.AddMemberWithValue("LC_VERSION_MIN_IPHONEOS", LC_VERSION_MIN_IPHONEOS); + cmdTypeBuilder.AddMemberWithValue("LC_FUNCTION_STARTS", LC_FUNCTION_STARTS); + cmdTypeBuilder.AddMemberWithValue("LC_DYLD_ENVIRONMENT", LC_DYLD_ENVIRONMENT); + cmdTypeBuilder.AddMemberWithValue("LC_MAIN", LC_MAIN); // (0x28 | LC_REQ_DYLD) + cmdTypeBuilder.AddMemberWithValue("LC_DATA_IN_CODE", LC_DATA_IN_CODE); + cmdTypeBuilder.AddMemberWithValue("LC_SOURCE_VERSION", LC_SOURCE_VERSION); + cmdTypeBuilder.AddMemberWithValue("LC_DYLIB_CODE_SIGN_DRS", LC_DYLIB_CODE_SIGN_DRS); + cmdTypeBuilder.AddMemberWithValue("LC_ENCRYPTION_INFO_64", _LC_ENCRYPTION_INFO_64); + cmdTypeBuilder.AddMemberWithValue("LC_LINKER_OPTION", _LC_LINKER_OPTION); + cmdTypeBuilder.AddMemberWithValue("LC_LINKER_OPTIMIZATION_HINT", _LC_LINKER_OPTIMIZATION_HINT); + cmdTypeBuilder.AddMemberWithValue("LC_VERSION_MIN_TVOS", _LC_VERSION_MIN_TVOS); + cmdTypeBuilder.AddMemberWithValue("LC_VERSION_MIN_WATCHOS", LC_VERSION_MIN_WATCHOS); + cmdTypeBuilder.AddMemberWithValue("LC_NOTE", LC_NOTE); + cmdTypeBuilder.AddMemberWithValue("LC_BUILD_VERSION", LC_BUILD_VERSION); + cmdTypeBuilder.AddMemberWithValue("LC_DYLD_EXPORTS_TRIE", LC_DYLD_EXPORTS_TRIE); + cmdTypeBuilder.AddMemberWithValue("LC_DYLD_CHAINED_FIXUPS", LC_DYLD_CHAINED_FIXUPS); + cmdTypeBuilder.AddMemberWithValue("LC_FILESET_ENTRY", LC_FILESET_ENTRY); + Ref<Enumeration> cmdTypeEnum = cmdTypeBuilder.Finalize(); + + Ref<Type> cmdTypeEnumType = Type::EnumerationType(nullptr, cmdTypeEnum, 4, false); + std::string cmdTypeEnumName = "load_command_type_t"; + std::string cmdTypeEnumId = Type::GenerateAutoTypeId("macho", cmdTypeEnumName); + DefineType(cmdTypeEnumId, cmdTypeEnumName, cmdTypeEnumType); + + StructureBuilder loadCommandBuilder; + loadCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + loadCommandBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + Ref<Structure> loadCommandStruct = loadCommandBuilder.Finalize(); + QualifiedName loadCommandName = std::string("load_command"); + std::string loadCommandTypeId = Type::GenerateAutoTypeId("macho", loadCommandName); + Ref<Type> loadCommandType = Type::StructureType(loadCommandStruct); + DefineType(loadCommandTypeId, loadCommandName, loadCommandType); + + EnumerationBuilder protTypeBuilder; + protTypeBuilder.AddMemberWithValue("VM_PROT_NONE", MACHO_VM_PROT_NONE); + protTypeBuilder.AddMemberWithValue("VM_PROT_READ", MACHO_VM_PROT_READ); + protTypeBuilder.AddMemberWithValue("VM_PROT_WRITE", MACHO_VM_PROT_WRITE); + protTypeBuilder.AddMemberWithValue("VM_PROT_EXECUTE", MACHO_VM_PROT_EXECUTE); + // protTypeBuilder.AddMemberWithValue("VM_PROT_DEFAULT", MACHO_VM_PROT_DEFAULT); + // protTypeBuilder.AddMemberWithValue("VM_PROT_ALL", MACHO_VM_PROT_ALL); + protTypeBuilder.AddMemberWithValue("VM_PROT_NO_CHANGE", MACHO_VM_PROT_NO_CHANGE); + protTypeBuilder.AddMemberWithValue("VM_PROT_COPY_OR_WANTS_COPY", MACHO_VM_PROT_COPY); + // protTypeBuilder.AddMemberWithValue("VM_PROT_WANTS_COPY", MACHO_VM_PROT_WANTS_COPY); + Ref<Enumeration> protTypeEnum = protTypeBuilder.Finalize(); + + Ref<Type> protTypeEnumType = Type::EnumerationType(nullptr, protTypeEnum, 4, false); + std::string protTypeEnumName = "vm_prot_t"; + std::string protTypeEnumId = Type::GenerateAutoTypeId("macho", protTypeEnumName); + DefineType(protTypeEnumId, protTypeEnumName, protTypeEnumType); + + EnumerationBuilder segFlagsTypeBuilder; + segFlagsTypeBuilder.AddMemberWithValue("SG_HIGHVM", SG_HIGHVM); + segFlagsTypeBuilder.AddMemberWithValue("SG_FVMLIB", SG_FVMLIB); + segFlagsTypeBuilder.AddMemberWithValue("SG_NORELOC", SG_NORELOC); + segFlagsTypeBuilder.AddMemberWithValue("SG_PROTECTED_VERSION_1", SG_PROTECTED_VERSION_1); + Ref<Enumeration> segFlagsTypeEnum = segFlagsTypeBuilder.Finalize(); + + Ref<Type> segFlagsTypeEnumType = Type::EnumerationType(nullptr, segFlagsTypeEnum, 4, false); + std::string segFlagsTypeEnumName = "sg_flags_t"; + std::string segFlagsTypeEnumId = Type::GenerateAutoTypeId("macho", segFlagsTypeEnumName); + DefineType(segFlagsTypeEnumId, segFlagsTypeEnumName, segFlagsTypeEnumType); + + StructureBuilder loadSegmentCommandBuilder; + loadSegmentCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + loadSegmentCommandBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + loadSegmentCommandBuilder.AddMember(Type::ArrayType(Type::IntegerType(1, true), 16), "segname"); + loadSegmentCommandBuilder.AddMember(Type::IntegerType(4, false), "vmaddr"); + loadSegmentCommandBuilder.AddMember(Type::IntegerType(4, false), "vmsize"); + loadSegmentCommandBuilder.AddMember(Type::IntegerType(4, false), "fileoff"); + loadSegmentCommandBuilder.AddMember(Type::IntegerType(4, false), "filesize"); + loadSegmentCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("vm_prot_t")), "maxprot"); + loadSegmentCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("vm_prot_t")), "initprot"); + loadSegmentCommandBuilder.AddMember(Type::IntegerType(4, false), "nsects"); + loadSegmentCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("sg_flags_t")), "flags"); + Ref<Structure> loadSegmentCommandStruct = loadSegmentCommandBuilder.Finalize(); + QualifiedName loadSegmentCommandName = std::string("segment_command"); + std::string loadSegmentCommandTypeId = Type::GenerateAutoTypeId("macho", loadSegmentCommandName); + Ref<Type> loadSegmentCommandType = Type::StructureType(loadSegmentCommandStruct); + DefineType(loadSegmentCommandTypeId, loadSegmentCommandName, loadSegmentCommandType); + + StructureBuilder loadSegmentCommand64Builder; + loadSegmentCommand64Builder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + loadSegmentCommand64Builder.AddMember(Type::IntegerType(4, false), "cmdsize"); + loadSegmentCommand64Builder.AddMember(Type::ArrayType(Type::IntegerType(1, true), 16), "segname"); + loadSegmentCommand64Builder.AddMember(Type::IntegerType(8, false), "vmaddr"); + loadSegmentCommand64Builder.AddMember(Type::IntegerType(8, false), "vmsize"); + loadSegmentCommand64Builder.AddMember(Type::IntegerType(8, false), "fileoff"); + loadSegmentCommand64Builder.AddMember(Type::IntegerType(8, false), "filesize"); + loadSegmentCommand64Builder.AddMember(Type::NamedType(this, QualifiedName("vm_prot_t")), "maxprot"); + loadSegmentCommand64Builder.AddMember(Type::NamedType(this, QualifiedName("vm_prot_t")), "initprot"); + loadSegmentCommand64Builder.AddMember(Type::IntegerType(4, false), "nsects"); + loadSegmentCommand64Builder.AddMember(Type::NamedType(this, QualifiedName("sg_flags_t")), "flags"); + Ref<Structure> loadSegmentCommand64Struct = loadSegmentCommand64Builder.Finalize(); + QualifiedName loadSegment64CommandName = std::string("segment_command_64"); + std::string loadSegment64CommandTypeId = Type::GenerateAutoTypeId("macho", loadSegment64CommandName); + Ref<Type> loadSegment64CommandType = Type::StructureType(loadSegmentCommand64Struct); + DefineType(loadSegment64CommandTypeId, loadSegment64CommandName, loadSegment64CommandType); + + StructureBuilder sectionBuilder; + sectionBuilder.AddMember(Type::ArrayType(Type::IntegerType(1, true), 16), "sectname"); + sectionBuilder.AddMember(Type::ArrayType(Type::IntegerType(1, true), 16), "segname"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "addr"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "size"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "offset"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "align"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "reloff"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "nreloc"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "flags"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "reserved1"); + sectionBuilder.AddMember(Type::IntegerType(4, false), "reserved2"); + Ref<Structure> sectionStruct = sectionBuilder.Finalize(); + QualifiedName sectionName = std::string("section"); + std::string sectionTypeId = Type::GenerateAutoTypeId("macho", sectionName); + Ref<Type> sectionType = Type::StructureType(sectionStruct); + DefineType(sectionTypeId, sectionName, sectionType); + + StructureBuilder section64Builder; + section64Builder.AddMember(Type::ArrayType(Type::IntegerType(1, true), 16), "sectname"); + section64Builder.AddMember(Type::ArrayType(Type::IntegerType(1, true), 16), "segname"); + section64Builder.AddMember(Type::IntegerType(8, false), "addr"); + section64Builder.AddMember(Type::IntegerType(8, false), "size"); + section64Builder.AddMember(Type::IntegerType(4, false), "offset"); + section64Builder.AddMember(Type::IntegerType(4, false), "align"); + section64Builder.AddMember(Type::IntegerType(4, false), "reloff"); + section64Builder.AddMember(Type::IntegerType(4, false), "nreloc"); + section64Builder.AddMember(Type::IntegerType(4, false), "flags"); + section64Builder.AddMember(Type::IntegerType(4, false), "reserved1"); + section64Builder.AddMember(Type::IntegerType(4, false), "reserved2"); + section64Builder.AddMember(Type::IntegerType(4, false), "reserved3"); + Ref<Structure> section64Struct = section64Builder.Finalize(); + QualifiedName section64Name = std::string("section_64"); + std::string section64TypeId = Type::GenerateAutoTypeId("macho", section64Name); + Ref<Type> section64Type = Type::StructureType(section64Struct); + DefineType(section64TypeId, section64Name, section64Type); + + StructureBuilder symtabBuilder; + symtabBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + symtabBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + symtabBuilder.AddMember(Type::IntegerType(4, false), "symoff"); + symtabBuilder.AddMember(Type::IntegerType(4, false), "nsyms"); + symtabBuilder.AddMember(Type::IntegerType(4, false), "stroff"); + symtabBuilder.AddMember(Type::IntegerType(4, false), "strsize"); + Ref<Structure> symtabStruct = symtabBuilder.Finalize(); + QualifiedName symtabName = std::string("symtab"); + std::string symtabTypeId = Type::GenerateAutoTypeId("macho", symtabName); + Ref<Type> symtabType = Type::StructureType(symtabStruct); + DefineType(symtabTypeId, symtabName, symtabType); + + StructureBuilder dynsymtabBuilder; + dynsymtabBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "ilocalsym"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nlocalsym"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "iextdefsym"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nextdefsym"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "iundefsym"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nundefsym"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "tocoff"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "ntoc"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "modtaboff"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nmodtab"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "extrefsymoff"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nextrefsyms"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "indirectsymoff"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nindirectsyms"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "extreloff"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nextrel"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "locreloff"); + dynsymtabBuilder.AddMember(Type::IntegerType(4, false), "nlocrel"); + Ref<Structure> dynsymtabStruct = dynsymtabBuilder.Finalize(); + QualifiedName dynsymtabName = std::string("dysymtab"); + std::string dynsymtabTypeId = Type::GenerateAutoTypeId("macho", dynsymtabName); + Ref<Type> dynsymtabType = Type::StructureType(dynsymtabStruct); + DefineType(dynsymtabTypeId, dynsymtabName, dynsymtabType); + + StructureBuilder uuidBuilder; + uuidBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + uuidBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + uuidBuilder.AddMember(Type::ArrayType(Type::IntegerType(1, false), 16), "uuid"); + Ref<Structure> uuidStruct = uuidBuilder.Finalize(); + QualifiedName uuidName = std::string("uuid"); + std::string uuidTypeId = Type::GenerateAutoTypeId("macho", uuidName); + Ref<Type> uuidType = Type::StructureType(uuidStruct); + DefineType(uuidTypeId, uuidName, uuidType); + + StructureBuilder linkeditDataBuilder; + linkeditDataBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + linkeditDataBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + linkeditDataBuilder.AddMember(Type::IntegerType(4, false), "dataoff"); + linkeditDataBuilder.AddMember(Type::IntegerType(4, false), "datasize"); + Ref<Structure> linkeditDataStruct = linkeditDataBuilder.Finalize(); + QualifiedName linkeditDataName = std::string("linkedit_data"); + std::string linkeditDataTypeId = Type::GenerateAutoTypeId("macho", linkeditDataName); + Ref<Type> linkeditDataType = Type::StructureType(linkeditDataStruct); + DefineType(linkeditDataTypeId, linkeditDataName, linkeditDataType); + + StructureBuilder encryptionInfoBuilder; + encryptionInfoBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + encryptionInfoBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + encryptionInfoBuilder.AddMember(Type::IntegerType(4, false), "cryptoff"); + encryptionInfoBuilder.AddMember(Type::IntegerType(4, false), "cryptsize"); + encryptionInfoBuilder.AddMember(Type::IntegerType(4, false), "cryptid"); + Ref<Structure> encryptionInfoStruct = encryptionInfoBuilder.Finalize(); + QualifiedName encryptionInfoName = std::string("encryption_info"); + std::string encryptionInfoTypeId = Type::GenerateAutoTypeId("macho", encryptionInfoName); + Ref<Type> encryptionInfoType = Type::StructureType(encryptionInfoStruct); + DefineType(encryptionInfoTypeId, encryptionInfoName, encryptionInfoType); + + StructureBuilder versionMinBuilder; + versionMinBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + versionMinBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + versionMinBuilder.AddMember(Type::IntegerType(4, false), "version"); + versionMinBuilder.AddMember(Type::IntegerType(4, false), "sdk"); + Ref<Structure> versionMinStruct = versionMinBuilder.Finalize(); + QualifiedName versionMinName = std::string("version_min"); + std::string versionMinTypeId = Type::GenerateAutoTypeId("macho", versionMinName); + Ref<Type> versionMinType = Type::StructureType(versionMinStruct); + DefineType(versionMinTypeId, versionMinName, versionMinType); + + StructureBuilder dyldInfoBuilder; + dyldInfoBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "rebase_off"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "rebase_size"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "bind_off"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "bind_size"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "weak_bind_off"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "weak_bind_size"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "lazy_bind_off"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "lazy_bind_size"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "export_off"); + dyldInfoBuilder.AddMember(Type::IntegerType(4, false), "export_size"); + Ref<Structure> dyldInfoStruct = dyldInfoBuilder.Finalize(); + QualifiedName dyldInfoName = std::string("dyld_info"); + std::string dyldInfoTypeId = Type::GenerateAutoTypeId("macho", dyldInfoName); + Ref<Type> dyldInfoType = Type::StructureType(dyldInfoStruct); + DefineType(dyldInfoTypeId, dyldInfoName, dyldInfoType); + + StructureBuilder dylibBuilder; + dylibBuilder.AddMember(Type::IntegerType(4, false), "name"); + dylibBuilder.AddMember(Type::IntegerType(4, false), "timestamp"); + dylibBuilder.AddMember(Type::IntegerType(4, false), "current_version"); + dylibBuilder.AddMember(Type::IntegerType(4, false), "compatibility_version"); + Ref<Structure> dylibStruct = dylibBuilder.Finalize(); + QualifiedName dylibName = std::string("dylib"); + std::string dylibTypeId = Type::GenerateAutoTypeId("macho", dylibName); + Ref<Type> dylibType = Type::StructureType(dylibStruct); + DefineType(dylibTypeId, dylibName, dylibType); + + StructureBuilder dylibCommandBuilder; + dylibCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + dylibCommandBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + dylibCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("dylib")), "dylib"); + Ref<Structure> dylibCommandStruct = dylibCommandBuilder.Finalize(); + QualifiedName dylibCommandName = std::string("dylib_command"); + std::string dylibCommandTypeId = Type::GenerateAutoTypeId("macho", dylibCommandName); + Ref<Type> dylibCommandType = Type::StructureType(dylibCommandStruct); + DefineType(dylibCommandTypeId, dylibCommandName, dylibCommandType); + + StructureBuilder filesetEntryCommandBuilder; + filesetEntryCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + filesetEntryCommandBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + filesetEntryCommandBuilder.AddMember(Type::IntegerType(8, false), "vmaddr"); + filesetEntryCommandBuilder.AddMember(Type::IntegerType(8, false), "fileoff"); + filesetEntryCommandBuilder.AddMember(Type::IntegerType(4, false), "entry_id"); + filesetEntryCommandBuilder.AddMember(Type::IntegerType(4, false), "reserved"); + Ref<Structure> filesetEntryCommandStruct = filesetEntryCommandBuilder.Finalize(); + QualifiedName filesetEntryCommandName = std::string("fileset_entry_command"); + std::string filesetEntryCommandTypeId = Type::GenerateAutoTypeId("macho", filesetEntryCommandName); + Ref<Type> filesetEntryCommandType = Type::StructureType(filesetEntryCommandStruct); + DefineType(filesetEntryCommandTypeId, filesetEntryCommandName, filesetEntryCommandType); + + StructureBuilder unixThreadCommandBuilder; + unixThreadCommandBuilder.AddMember(Type::NamedType(this, QualifiedName("load_command_type_t")), "cmd"); + unixThreadCommandBuilder.AddMember(Type::IntegerType(4, false), "cmdsize"); + unixThreadCommandBuilder.AddMember(Type::IntegerType(4, false), "flavor"); + unixThreadCommandBuilder.AddMember(Type::IntegerType(4, false), "count"); + // The 'state' field is intentionally ignored. + Ref<Structure> unixThreadCommandStruct = unixThreadCommandBuilder.Finalize(); + QualifiedName unixThreadCommandName = std::string("unix_thread_command"); + std::string unixThreadCommandTypeId = Type::GenerateAutoTypeId("macho", unixThreadCommandName); + Ref<Type> unixThreadCommandType = Type::StructureType(unixThreadCommandStruct); + DefineType(unixThreadCommandTypeId, unixThreadCommandName, unixThreadCommandType); + + std::vector<KernelCacheCore::MemoryRegion> regionsMappedIntoMemory; + if (auto metadata = KernelCacheCore::KernelCacheMetadata::LoadFromView(GetParentView())) + { + for (const auto& image : metadata->LoadedImages()) + { + auto header = KernelCacheCore::KernelCache::LoadHeaderForAddress(this, image.headerFileLocation, image.installName); + if (!header) + { + LogError("Failed to load header for image %s", image.installName.c_str()); + return false; + } + if (!KernelCacheCore::KernelCache::InitializeSegmentsForHeader(this, *header, image)) + { + LogError("Failed to initialize segments for image %s", image.installName.c_str()); + return false; + } + KernelCacheCore::KernelCache::InitializeHeader(this, *header); + } + } + + // Technically the header is executable, but there shouldn't reasonably be code in the header. + AddAutoSegment(textSegOffset, sizeofcmds + 0x20, 0, sizeofcmds + 0x20, SegmentReadable); + AddAutoSection("kernelcache_header", textSegOffset, sizeofcmds + 0x20, ReadOnlyDataSectionSemantics); + + if (m_parseOnly) + return true; + + DefineDataVariable(textSegOffset, Type::NamedType(this, QualifiedName("mach_header_64"))); + DefineAutoSymbol( + new Symbol(DataSymbol, "kernelcache_header", textSegOffset, LocalBinding)); + + try + { + reader.Seek(sizeof(mach_header_64)); + size_t sectionNum = 0; + for (size_t i = 0; i < ncmds; i++) + { + load_command load; + uint64_t curOffset = reader.GetOffset(); + load.cmd = reader.Read32(); + load.cmdsize = reader.Read32(); + uint64_t nextOffset = curOffset + load.cmdsize; + switch (load.cmd) + { + case LC_SEGMENT: + { + DefineDataVariable(reader.GetOffset() + textSegOffset, Type::NamedType(this, QualifiedName("segment_command"))); + reader.SeekRelative(5 * 8); + size_t numSections = reader.Read32(); + reader.SeekRelative(4); + for (size_t j = 0; j < numSections; j++) + { + DefineDataVariable( + reader.GetOffset() + textSegOffset, Type::NamedType(this, QualifiedName("section"))); + DefineAutoSymbol(new Symbol(DataSymbol, + "__macho_section::kernelcache[" + std::to_string(sectionNum++) + "]", + reader.GetOffset() + textSegOffset, LocalBinding)); + reader.SeekRelative((8 * 8) + 4); + } + break; + } + case LC_SEGMENT_64: + { + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("segment_command_64"))); + reader.SeekRelative(7 * 8); + size_t numSections = reader.Read32(); + reader.SeekRelative(4); + for (size_t j = 0; j < numSections; j++) + { + DefineDataVariable( + reader.GetOffset() + textSegOffset, Type::NamedType(this, QualifiedName("section_64"))); + DefineAutoSymbol(new Symbol(DataSymbol, + "__macho_section_64::kernelcache[" + std::to_string(sectionNum++) + "]", + reader.GetOffset() + textSegOffset, LocalBinding)); + reader.SeekRelative(10 * 8); + } + break; + } + case LC_SYMTAB: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("symtab"))); + break; + case LC_DYSYMTAB: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("dysymtab"))); + break; + case LC_UUID: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("uuid"))); + break; + case LC_ID_DYLIB: + case LC_LOAD_DYLIB: + case LC_REEXPORT_DYLIB: + case LC_LOAD_WEAK_DYLIB: + case LC_LOAD_UPWARD_DYLIB: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("dylib_command"))); + if (load.cmdsize - 24 <= 150) + DefineDataVariable( + curOffset + textSegOffset + 24, Type::ArrayType(Type::IntegerType(1, true), load.cmdsize - 24)); + break; + case LC_CODE_SIGNATURE: + case LC_SEGMENT_SPLIT_INFO: + case LC_FUNCTION_STARTS: + case LC_DATA_IN_CODE: + case LC_DYLIB_CODE_SIGN_DRS: + case LC_DYLD_EXPORTS_TRIE: + case LC_DYLD_CHAINED_FIXUPS: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("linkedit_data"))); + break; + case LC_ENCRYPTION_INFO: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("encryption_info"))); + break; + case LC_VERSION_MIN_MACOSX: + case LC_VERSION_MIN_IPHONEOS: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("version_min"))); + break; + case LC_DYLD_INFO: + case LC_DYLD_INFO_ONLY: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("dyld_info"))); + break; + case LC_FILESET_ENTRY: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("fileset_entry_command"))); + break; + case LC_UNIXTHREAD: + { + DefineDataVariable( + curOffset + textSegOffset, Type::NamedType(this, QualifiedName("unix_thread_command"))); + reader.SeekRelative(4); + uint32_t count = reader.Read32(); + DefineDataVariable(reader.GetOffset() + textSegOffset, Type::ArrayType(Type::IntegerType(8, true), count)); + break; + } + default: + DefineDataVariable(curOffset + textSegOffset, Type::NamedType(this, QualifiedName("load_command"))); + break; + } + + DefineAutoSymbol(new Symbol(DataSymbol, + "__macho_load_command::kernelcache[" + std::to_string(i) + "]", curOffset + textSegOffset, + LocalBinding)); + reader.Seek(nextOffset); + } + } + catch (ReadException&) + { + LogError("Error when applying Mach-O header types at %" PRIx64, textSegOffset); + } + + return true; +} + + +KCViewType::KCViewType() : BinaryViewType(KC_VIEW_NAME, KC_VIEW_NAME) +{ +} + +BinaryNinja::Ref<BinaryNinja::BinaryView> KCViewType::Create(BinaryNinja::BinaryView* data) +{ + uint32_t magic; + data->Read(&magic, data->GetStart(), 4); + if (magic != MH_CIGAM_64 && magic != MH_MAGIC_64) // FIXME 32 bit + { + uint32_t im4pMagic; + data->Read(&im4pMagic, data->GetStart() + 0x8, 4); + if (im4pMagic == 0x50344d49) // P4MI + { + auto img4 = Transform::GetByName("IMG4-Unencrypted"); + + DataBuffer img4Payload; + img4->Decode(data->ReadBuffer(data->GetStart(), data->GetLength()), img4Payload); + + DataBuffer machOPayload; + uint32_t magic = ((uint32_t*)img4Payload.GetData())[0]; + if (magic == FAT_MAGIC_64 || magic == MH_MAGIC_64 || magic == MH_MAGIC + || magic == MH_CIGAM_64 || magic == MH_CIGAM ) + { + machOPayload = img4Payload; + } + else if (strncmp((char*)img4Payload.GetData(), "bvx2", 4) == 0) + { + auto lzfse = Transform::GetByName("LZFSE"); + if (lzfse) + lzfse->Decode(img4Payload, machOPayload); + } + else + { +#ifdef COMPRESSION_DEBUG + LogError("Unknown compression type in IMG4 Payload, writing img4 payload to RAW view for debug purposes."); + LogError("KernelCache parsing will now fail to proceed."); + data->WriteBuffer(0, img4Payload); + return new KCView(KC_VIEW_NAME, data, false); +#else + LogError("Unknown compression type in IMG4 Payload, unable to proceed."); + LogError("You can manually extract the kernelcache using `kerneldec`,`ipsw`, or other tools.") + return nullptr; +#endif + } + + if (machOPayload.GetLength() == 0) + { +#ifdef COMPRESSION_DEBUG + LogError("Failed to perform extraction on IMG4 Payload, writing img4 payload to RAW view for debug purposes."); + LogError("KernelCache parsing will now fail to proceed."); + data->WriteBuffer(0, img4Payload); + return new KCView(KC_VIEW_NAME, data, false); +#else + return nullptr; +#endif + } + + uint32_t machoMagic = ((uint32_t*)machOPayload.GetData())[0]; + if (machoMagic == FAT_MAGIC_64) + { + DataBuffer output = machOPayload.GetSlice(0x1c, machOPayload.GetLength()-0x1c); + data->WriteBuffer(0, output); + } + else if (machoMagic == MH_MAGIC_64 || machoMagic == MH_MAGIC || machoMagic == MH_CIGAM_64 || machoMagic == MH_CIGAM) + { + data->WriteBuffer(0, machOPayload); + } + else + { +#ifdef COMPRESSION_DEBUG + LogError("Unknown Mach-O magic in IMG4 Payload, writing img4 payload to RAW view for debug purposes."); + LogError("KernelCache parsing will now fail to proceed."); + data->WriteBuffer(0, machOPayload); + return new KCView(KC_VIEW_NAME, data, false); +#else + return nullptr; +#endif + } + + return new KCView(KC_VIEW_NAME, data, false); + } + + return nullptr; + } + + return new KCView(KC_VIEW_NAME, data, false); +} + + +Ref<Settings> KCViewType::GetLoadSettingsForData(BinaryView* data) +{ + Ref<BinaryView> viewRef = Parse(data); + if (!viewRef || !viewRef->Init()) + { + LogWarn("Failed to initialize view of type '%s'. Generating default load settings.", GetName().c_str()); + viewRef = data; + } + + Ref<Settings> settings = GetDefaultLoadSettingsForData(viewRef); + + // specify default load settings that can be overridden + std::vector<std::string> overrides = {"loader.imageBase", "loader.platform"}; + settings->UpdateProperty("loader.imageBase", "message", "Note: File indicates image is not relocatable."); + + for (const auto& override : overrides) + { + if (settings->Contains(override)) + settings->UpdateProperty(override, "readOnly", false); + } + + // Merge existing load settings if they exist. This allows for the selection of a specific object file from a Mach-O + // Universal file. The 'Universal' BinaryViewType generates a schema with 'loader.universal.architectures'. This + // schema contains an appropriate 'Mach-O' load schema for selecting a specific object file. The embedded schema + // contains 'loader.macho.universalImageOffset'. + Ref<Settings> loadSettings = viewRef->GetLoadSettings(GetName()); + if (loadSettings && !loadSettings->IsEmpty()) + settings->DeserializeSchema(loadSettings->SerializeSchema()); + + return settings; +} + + +BinaryNinja::Ref<BinaryNinja::BinaryView> KCViewType::Parse(BinaryNinja::BinaryView* data) +{ + uint32_t magic; + data->Read(&magic, data->GetStart(), 4); + if (magic != MH_CIGAM_64 && magic != MH_MAGIC_64) // FIXME 32 bit + { + uint32_t im4pMagic; + data->Read(&im4pMagic, data->GetStart() + 0x8, 4); + if (im4pMagic == 0x50344d49) // P4MI + { + auto img4 = Transform::GetByName("IMG4-Unencrypted"); + auto lzfse = Transform::GetByName("LZFSE"); + + DataBuffer img4Payload; + img4->Decode(data->ReadBuffer(data->GetStart(), data->GetLength()), img4Payload); + DataBuffer machOPayload; + lzfse->Decode(img4Payload, machOPayload); + + uint32_t magic = ((uint32_t*)machOPayload.GetData())[0]; + auto id = data->BeginUndoActions(); + if (magic == FAT_MAGIC_64) + { + DataBuffer output = machOPayload.GetSlice(0x1c, machOPayload.GetLength()-0x1c); + data->WriteBuffer(0, output); + } + else + { + data->WriteBuffer(0, machOPayload); + } + data->ForgetUndoActions(id); + return new KCView(KC_VIEW_NAME, data, true); + } + + return nullptr; + } + + return new KCView(KC_VIEW_NAME, data, true); +} + +bool KCViewType::IsTypeValidForData(BinaryNinja::BinaryView* data) +{ + if (!data) + return false; + + uint32_t magic; + data->Read(&magic, data->GetStart(), 4); + + if (magic != MH_CIGAM_64 && magic != MH_MAGIC_64) // FIXME 32 bit + { + uint32_t im4pMagic; + data->Read(&im4pMagic, data->GetStart() + 0x8, 4); + if (im4pMagic == 0x50344d49) // P4MI + { + auto img4 = Transform::GetByName("IMG4-Unencrypted"); + auto lzfse = Transform::GetByName("LZFSE"); + + DataBuffer img4Payload; + img4->Decode(data->ReadBuffer(data->GetStart(), data->GetLength()), img4Payload); + DataBuffer machOPayload; + lzfse->Decode(img4Payload, machOPayload); + + uint32_t magic = ((uint32_t*)machOPayload.GetData())[0]; + if (magic == FAT_MAGIC_64 || magic == MH_CIGAM_64 || magic == MH_MAGIC_64) + return true; + + return false; + } + + return false; + } + + uint32_t fileType; + data->Read(&fileType, data->GetStart() + 0xc, 4); + if (fileType != MH_FILESET) + { + return false; + } + + return true; +} + +extern "C" { + void InitKCViewType() + { + static KCViewType type; + BinaryViewType::Register(&type); + g_kcViewType = &type; + } +} + diff --git a/view/kernelcache/core/KCView.h b/view/kernelcache/core/KCView.h new file mode 100644 index 00000000..c78833c0 --- /dev/null +++ b/view/kernelcache/core/KCView.h @@ -0,0 +1,46 @@ +// +// Created by kat on 5/23/23. +// + +#ifndef KERNELCACHE_KCVIEW_H +#define KERNELCACHE_KCVIEW_H + +#include <binaryninjaapi.h> + +class KCView : public BinaryNinja::BinaryView { + bool m_parseOnly; +public: + + KCView(const std::string &typeName, BinaryView *data, bool parseOnly = false); + + ~KCView() override; + + bool Init() override; +}; + + +class KCViewType : public BinaryNinja::BinaryViewType { + +public: + KCViewType(); + + BinaryNinja::Ref<BinaryNinja::BinaryView> Create(BinaryNinja::BinaryView *data) override; + + BinaryNinja::Ref<BinaryNinja::BinaryView> Parse(BinaryNinja::BinaryView *data) override; + + bool IsTypeValidForData(BinaryNinja::BinaryView *data) override; + + bool IsDeprecated() override { return false; } + + BinaryNinja::Ref<BinaryNinja::Settings> GetLoadSettingsForData(BinaryNinja::BinaryView *data) override; +}; + +#ifdef __cplusplus +extern "C" { +#endif + void InitKCViewType(); +#ifdef __cplusplus +}; +#endif + +#endif //KERNELCACHE_KCVIEW_H diff --git a/view/kernelcache/core/KernelCache.cpp b/view/kernelcache/core/KernelCache.cpp new file mode 100644 index 00000000..41577c0b --- /dev/null +++ b/view/kernelcache/core/KernelCache.cpp @@ -0,0 +1,2709 @@ +// +// Created by kat on 5/19/23. +// + +#include "binaryninjaapi.h" + +/* --- + * This is the primary image loader logic for Kernelcaches. + * */ + +#include "KernelCache.h" +#include <filesystem> +#include <utility> +#include <fcntl.h> +#include <memory> +#include <chrono> +#include <thread> + + +using namespace BinaryNinja; +using namespace KernelCacheCore; + +namespace KernelCacheCore { + + + +#ifdef _MSC_VER + +int count_trailing_zeros(uint64_t value) { + unsigned long index; // 32-bit long on Windows + if (_BitScanForward64(&index, value)) { + return index; + } else { + return 64; // If the value is 0, return 64. + } +} +#else +int count_trailing_zeros(uint64_t value) { + return value == 0 ? 64 : __builtin_ctzll(value); +} +#endif + +// State that does not change after `PerformInitialLoad`. +struct KernelCache::CacheInfo : + public MetadataSerializable<KernelCache::CacheInfo, std::optional<KernelCache::CacheInfo>> +{ + // std::unordered_map<uint64_t, KernelCacheMachOHeader> headers; + std::vector<KernelCacheImage> images; + std::unordered_map<std::string, uint64_t> imageStarts; + + KernelCacheFormat cacheFormat = FilesetCacheFormat; + +#ifndef NDEBUG + void Verify() const; +#endif + + uint64_t BaseAddress() const; + + void Store(SerializationContext&) const; + static std::optional<KernelCache::CacheInfo> Load(DeserializationContext&); +}; + +struct State : public MetadataSerializable<State> +{ + std::unordered_map<uint64_t, KernelCacheImage> loadedImages; + // Store only. Loading is done via `ModifiedState`. + void Store(SerializationContext&, std::optional<KCViewState> viewState) const; +}; + +struct KernelCache::ModifiedState : public State, public MetadataSerializable<KernelCache::ModifiedState> +{ + std::optional<KCViewState> viewState; + + using Base = MetadataSerializable<KernelCache::ModifiedState>; + using Base::AsMetadata; + using Base::LoadFromString; + + void Store(SerializationContext&) const; + static KernelCache::ModifiedState Load(DeserializationContext&); + static KernelCache::ModifiedState LoadAll(BinaryNinja::BinaryView*, const CacheInfo&); + + void Merge(KernelCache::ModifiedState&& other); +}; + +struct KernelCache::ViewSpecificState +{ + std::mutex typeLibraryMutex; + std::unordered_map<std::string, Ref<TypeLibrary>> typeLibraries; + + std::mutex viewOperationsThatInfluenceMetadataMutex; + + std::atomic<BNKCViewLoadProgress> progress; + + std::mutex cacheInfoMutex; + std::shared_ptr<const KernelCache::CacheInfo> cacheInfo; + + std::mutex stateMutex; + struct State state; + + std::atomic<KCViewState> viewState; + uint64_t savedModifications = 0; +}; + +namespace { + + std::shared_ptr<KernelCache::ViewSpecificState> ViewSpecificStateForId( + uint64_t viewIdentifier, bool insertIfNeeded = true) + { + static std::mutex viewSpecificStateMutex; + static std::unordered_map<uint64_t, std::weak_ptr<KernelCache::ViewSpecificState>> viewSpecificState; + + std::lock_guard lock(viewSpecificStateMutex); + + if (auto it = viewSpecificState.find(viewIdentifier); it != viewSpecificState.end()) + { + if (auto statePtr = it->second.lock()) + return statePtr; + } + + if (!insertIfNeeded) + return nullptr; + + auto statePtr = std::make_shared<KernelCache::ViewSpecificState>(); + viewSpecificState[viewIdentifier] = statePtr; + + // Prune entries for any views that are no longer in use. + for (auto it = viewSpecificState.begin(); it != viewSpecificState.end();) + { + if (it->second.expired()) + it = viewSpecificState.erase(it); + else + ++it; + } + + return statePtr; + } + + std::shared_ptr<KernelCache::ViewSpecificState> ViewSpecificStateForView(Ref<BinaryNinja::BinaryView> view) + { + return ViewSpecificStateForId(view->GetFile()->GetSessionId()); + } + + std::string base_name(std::string const& path) + { + return path.substr(path.find_last_of("/\\") + 1); + } + +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunused-function" + static int64_t readSLEB128(DataBuffer& buffer, size_t length, size_t& offset) + { + uint8_t cur; + int64_t value = 0; + size_t shift = 0; + while (offset < length) + { + cur = buffer[offset++]; + value |= (cur & 0x7f) << shift; + shift += 7; + if ((cur & 0x80) == 0) + break; + } + value = (value << (64 - shift)) >> (64 - shift); + return value; + } +#pragma clang diagnostic pop + + + static uint64_t readLEB128(DataBuffer& p, size_t end, size_t& offset) + { + uint64_t result = 0; + int bit = 0; + do + { + if (offset >= end) + return -1; + + uint64_t slice = p[offset] & 0x7f; + + if (bit > 63) + return -1; + else + { + result |= (slice << bit); + bit += 7; + } + } while (p[offset++] & 0x80); + return result; + } + + + uint64_t readValidULEB128(DataBuffer& buffer, size_t& cursor) + { + uint64_t value = readLEB128(buffer, buffer.GetLength(), cursor); + if ((int64_t)value == -1) + throw ReadException(); + return value; + } + +} // namespace + +void KernelCache::PerformInitialLoad(std::lock_guard<std::mutex>& lock) +{ + bool is64 = m_kcView->GetAddressSize() == 8; + + m_logger->LogInfo("Performing initial load of Kernel Cache"); + + m_viewSpecificState->progress = LoadProgressLoadingCaches; + + CacheInfo initialState; + initialState.cacheFormat = FilesetCacheFormat; + + m_viewSpecificState->progress = LoadProgressLoadingImages; + + // We have set up enough metadata to map VM now. + // Iterate load comands: + BinaryReader reader(m_kcView->GetParentView()); + + uint32_t magic = reader.Read32(); + if (magic != MH_CIGAM_64 && magic != MH_MAGIC_64) // FIXME 32 bit + { + m_logger->LogError("Invalid magic number in KernelCache"); + return; + } + uint32_t cpuType = reader.Read32(); + uint32_t cpuSubtype = reader.Read32(); + uint32_t fileType = reader.Read32(); + if (fileType != MH_FILESET) + { + m_logger->LogError("Invalid file type in KernelCache"); + return; + } + uint32_t ncmds = reader.Read32(); + uint32_t sizeofcmds = reader.Read32(); + uint32_t flags = reader.Read32(); + if ((cpuType & MachOABIMask) != MachOABI64) + { + m_logger->LogError("Invalid ABI in KernelCache. 32 bit not yet supported."); + return; + } + if (is64) + reader.SeekRelative(4); + + uint64_t off = reader.GetOffset(); + + while (ncmds--) + { + reader.Seek(off); + uint32_t cmd = reader.Read32(); + uint32_t cmdsize = reader.Read32(); + if (cmd == LC_FILESET_ENTRY) + { + // uint64_t vmAddr =reader.Read64(); + reader.SeekRelative(m_kcView->GetAddressSize()); + uint64_t fileOff = reader.Read64(); + uint32_t entryID = reader.Read32(); + reader.Seek(entryID + off); + std::string installName = reader.ReadCString(0x1000); + initialState.imageStarts[installName] = fileOff; + } + off += cmdsize; + } + + for (const auto& start : initialState.imageStarts) + { + try { + auto imageHeader = KernelCache::LoadHeaderForAddress(m_kcView, start.second, start.first); + if (imageHeader) + { + KernelCacheImage image; + image.installName = start.first; + image.headerFileLocation = start.second; + for (const auto& segment : imageHeader->segments) + { + char segName[17]; + memcpy(segName, segment.segname, 16); + segName[16] = 0; + MemoryRegion sectionRegion; + sectionRegion.prettyName = imageHeader.value().identifierPrefix + "::" + std::string(segName); + sectionRegion.start = segment.vmaddr; + sectionRegion.size = segment.vmsize; + sectionRegion.fileOffset = segment.fileoff; + uint32_t flags = 0; + if (segment.initprot & MACHO_VM_PROT_READ) + flags |= SegmentReadable; + if (segment.initprot & MACHO_VM_PROT_WRITE) + flags |= SegmentWritable; + if (segment.initprot & MACHO_VM_PROT_EXECUTE) + flags |= SegmentExecutable; + if (((segment.initprot & MACHO_VM_PROT_WRITE) == 0) && + ((segment.maxprot & MACHO_VM_PROT_WRITE) == 0)) + flags |= SegmentDenyWrite; + if (((segment.initprot & MACHO_VM_PROT_EXECUTE) == 0) && + ((segment.maxprot & MACHO_VM_PROT_EXECUTE) == 0)) + flags |= SegmentDenyExecute; + + // if we're positive we have an entry point for some reason, force the segment + // executable. this helps with kernel images. + for (auto &entryPoint : imageHeader->m_entryPoints) + if (segment.vmaddr <= entryPoint && (entryPoint < (segment.vmaddr + segment.filesize))) + flags |= SegmentExecutable; + + sectionRegion.flags = (BNSegmentFlag)flags; + image.regions.push_back(sectionRegion); + } + initialState.images.push_back(image); + } + else + { + m_logger->LogError("Failed to load Mach-O header for %s", start.first.c_str()); + } + } + catch (std::exception& ex) + { + m_logger->LogError("Failed to load Mach-O header for %s: %s", start.first.c_str(), ex.what()); + } + } + + m_cacheInfo = std::make_shared<CacheInfo>(std::move(initialState)); + m_modifiedState->viewState = KCViewStateLoaded; + SaveCacheInfoToKCView(lock); + SaveModifiedStateToKCView(lock); + + m_logger->LogDebug("Finished initial load of KernelCache"); + + m_viewSpecificState->progress = LoadProgressFinished; +} + +void KernelCache::DeserializeFromRawView(std::lock_guard<std::mutex>& lock) +{ + std::lock_guard cacheInfoLock(m_viewSpecificState->cacheInfoMutex); + if (m_viewSpecificState->cacheInfo) + { + m_cacheInfo = m_viewSpecificState->cacheInfo; + m_modifiedState = std::make_unique<ModifiedState>(); + m_metadataValid = true; + return; + } + + if (KernelCacheMetadata::ViewHasMetadata(m_kcView)) + { + auto metadata = KernelCacheMetadata::LoadFromView(m_kcView); + if (!metadata) + { + m_metadataValid = false; + m_logger->LogError("Failed to deserialize Shared Cache metadata"); + return; + } + + m_viewSpecificState->viewState = metadata->state->viewState.value_or(KCViewStateUnloaded); + m_viewSpecificState->state = std::move(*metadata->state); + m_viewSpecificState->cacheInfo = std::move(metadata->cacheInfo); + + m_cacheInfo = m_viewSpecificState->cacheInfo; + m_modifiedState = std::make_unique<ModifiedState>(); + m_metadataValid = true; + return; + } + + m_cacheInfo = nullptr; + m_modifiedState = std::make_unique<ModifiedState>(); + m_modifiedState->viewState = KCViewStateUnloaded; + m_metadataValid = true; +} + + +std::string to_hex_string(uint64_t value) +{ + std::stringstream ss; + ss << std::hex << value; + return ss.str(); +} + + +KernelCache::KernelCache(BinaryNinja::Ref<BinaryNinja::BinaryView> kcView) : m_kcView(kcView), + m_viewSpecificState(ViewSpecificStateForView(kcView)) +{ + std::lock_guard lock(m_mutex); + m_logger = LogRegistry::GetLogger("KernelCache", kcView->GetFile()->GetSessionId()); + if (kcView->GetTypeName() != KC_VIEW_NAME) + { + // Unreachable? + m_logger->LogError("Attempted to create KernelCache object from non-KernelCache view"); + return; + } + + INIT_KERNELCACHE_API_OBJECT() + DeserializeFromRawView(lock); + if (!m_metadataValid) + return; + + if (m_modifiedState->viewState.value_or(m_viewSpecificState->viewState) != KCViewStateUnloaded) + { + m_viewSpecificState->progress = LoadProgressFinished; + return; + } + + std::unique_lock viewOperationsLock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex); + + try { + PerformInitialLoad(lock); + } + catch (...) + { + m_logger->LogError("Failed to perform initial load of KernelCache"); + } +} + +KernelCache::~KernelCache() { +} + +KernelCache* KernelCache::GetFromKCView(BinaryNinja::Ref<BinaryNinja::BinaryView> kcView) +{ + if (kcView->GetTypeName() != KC_VIEW_NAME) + return nullptr; + try { + return new KernelCache(kcView); + } + catch (...) + { + return nullptr; + } +} + + +uint64_t KernelCache::FastGetImageCount(BinaryNinja::Ref<BinaryNinja::BinaryView> kcView) +{ + if (!kcView->GetParentView()) + return 0; + auto reader = BinaryReader(kcView->GetParentView()); + uint32_t magic = reader.Read32(); + if (magic != MH_CIGAM_64 && magic != MH_MAGIC_64) // FIXME 32 bit + { + return 0; + } + reader.Seek(0xc); + uint32_t fileType = reader.Read32(); + if (fileType != MH_FILESET) + { + return 0; + } + uint32_t ncmds = reader.Read32(); + + uint64_t imageCount = 0; + + uint64_t off = 0x20; // FIXME 32 bit + while (ncmds--) + { + reader.Seek(off); + uint32_t cmd = reader.Read32(); + uint32_t cmdsize = reader.Read32(); + if (cmd == LC_FILESET_ENTRY) + { + imageCount++; + } + off += cmdsize; + } + + return imageCount; +} + + +const std::unordered_map<std::string, uint64_t>& KernelCache::AllImageStarts() const +{ + return m_cacheInfo->imageStarts; +} + + +const std::unordered_map<uint64_t, KernelCacheMachOHeader> KernelCache::AllImageHeaders() const +{ + std::unordered_map<uint64_t, KernelCacheMachOHeader> headers; + for (const auto& start : m_cacheInfo->imageStarts) + { + auto header = LoadHeaderForAddress(m_kcView, start.second, start.first); + if (header) + { + headers[start.second] = *header; + } + } + return headers; +} + + +KCViewState KernelCache::ViewState() const +{ + return m_viewSpecificState->viewState; +} + +std::optional<uint64_t> KernelCache::GetImageStart(std::string installName) +{ + for (const auto& [name, start] : m_cacheInfo->imageStarts) + { + if (name == installName) + { + return start; + } + } + return {}; +} + +std::optional<KernelCacheMachOHeader> KernelCache::HeaderForVMAddress(uint64_t address) +{ + for (const auto& img : m_cacheInfo->images) + { + for (const auto& region : img.regions) + { + if (region.start <= address && region.start + region.size > address) + { + return LoadHeaderForAddress(m_kcView, img.headerFileLocation, img.installName); + } + } + } + return {}; +} + +std::optional<KernelCacheMachOHeader> KernelCache::HeaderForFileAddress(uint64_t address) +{ + for (const auto& img : m_cacheInfo->images) + { + for (const auto& region : img.regions) + { + if (region.fileOffset <= address && region.fileOffset + region.size > address) + { + return LoadHeaderForAddress(m_kcView, img.headerFileLocation, img.installName); + } + } + } + return {}; +} + +std::string KernelCache::NameForAddress(uint64_t address) +{ + if (auto header = HeaderForVMAddress(address)) + { + for (const auto& section : header->sections) + { + if (section.addr <= address && section.addr + section.size > address) + { + char sectionName[17]; + strncpy(sectionName, section.sectname, 16); + sectionName[16] = '\0'; + return header->identifierPrefix + "::" + sectionName; + } + } + } + return ""; +} + +std::string KernelCache::ImageNameForAddress(uint64_t address) +{ + if (auto header = HeaderForVMAddress(address)) + { + return header->identifierPrefix; + } + return ""; +} + +bool KernelCache::LoadImageContainingAddress(uint64_t address) +{ + std::lock_guard<std::mutex> lock(m_mutex); + return LoadImageContainingAddress(lock, address); +} + +bool KernelCache::LoadImageContainingAddress(std::lock_guard<std::mutex>& lock, uint64_t address) +{ + for (const auto& img : m_cacheInfo->images) + { + for (const auto& region : img.regions) + { + if (region.start <= address && region.start + region.size > address) + { + return LoadImageWithInstallName(lock, img.installName); + } + } + } + return false; +} + + +bool KernelCache::LoadImageWithInstallName(std::string installName) +{ + std::lock_guard<std::mutex> lock(m_mutex); + return LoadImageWithInstallName(lock, installName); +} + + +bool KernelCache::LoadImageWithInstallName(std::lock_guard<std::mutex>& lock, std::string installName) +{ + auto settings = m_kcView->GetLoadSettings(KC_VIEW_NAME); + + std::lock_guard viewSpecificStateLock(m_viewSpecificState->viewOperationsThatInfluenceMetadataMutex); + + m_logger->LogInfo("Loading image %s", installName.c_str()); + const KernelCacheImage* targetImage = nullptr; + + // FIXME at some point the logic func should be one with targetImage passed, and an installName wrapper can be added. + // In many use cases of this function we already have our target image. + for (auto& cacheImage : m_cacheInfo->images) + { + if (cacheImage.installName == installName) + { + targetImage = &cacheImage; + break; + } + } + const auto header = LoadHeaderForAddress(m_kcView, targetImage->headerFileLocation, targetImage->installName); + + if (!header) + { + return false; + } + + m_modifiedState->viewState = KCViewStateLoadedWithImages; + + InitializeSegmentsForHeader(m_kcView, *header, *targetImage); + + // Optimize relocations we just added en masse. + m_kcView->FinalizeNewSegments(); + + // FIXME Load TypeLibrary here when we have those for the kernel. + + m_modifiedState->loadedImages[targetImage->headerFileLocation] = *targetImage; + + SaveModifiedStateToKCView(lock); + + auto h = KernelCache::LoadHeaderForAddress(m_kcView, targetImage->headerFileLocation, installName); + if (!h.has_value()) + { + return false; + } + + KernelCache::InitializeHeader(m_kcView, *h); + + m_kcView->AddAnalysisOption("linearsweep"); + m_kcView->UpdateAnalysis(); + + return true; +} + +std::optional<KernelCacheMachOHeader> KernelCache::LoadHeaderForAddress(Ref<BinaryView> view, uint64_t address, std::string installName) +{ + KernelCacheMachOHeader header; + + header.installName = installName; + header.identifierPrefix = base_name(installName); + + std::string errorMsg; + // address is a Raw file offset + BinaryReader reader(view->GetParentView()); + reader.Seek(address); + + header.ident.magic = reader.Read32(); + + BNEndianness endianness; + if (header.ident.magic == MH_MAGIC || header.ident.magic == MH_MAGIC_64) + endianness = LittleEndian; + else if (header.ident.magic == MH_CIGAM || header.ident.magic == MH_CIGAM_64) + endianness = BigEndian; + else + { + return {}; + } + + reader.SetEndianness(endianness); + header.ident.cputype = reader.Read32(); + header.ident.cpusubtype = reader.Read32(); + header.ident.filetype = reader.Read32(); + header.ident.ncmds = reader.Read32(); + header.ident.sizeofcmds = reader.Read32(); + header.ident.flags = reader.Read32(); + if ((header.ident.cputype & MachOABIMask) == MachOABI64) // address size == 8 + { + header.ident.reserved = reader.Read32(); + } + header.loadCommandOffset = reader.GetOffset(); + + // Parse segment commands + try + { + for (size_t i = 0; i < header.ident.ncmds; i++) + { + // BNLogInfo("of 0x%llx", reader.GetOffset()); + load_command load; + segment_command_64 segment64; + section_64 sect; + memset(§, 0, sizeof(sect)); + size_t curOffset = reader.GetOffset(); + load.cmd = reader.Read32(); + load.cmdsize = reader.Read32(); + size_t nextOffset = curOffset + load.cmdsize; + if (load.cmdsize < sizeof(load_command)) + return {}; + + switch (load.cmd) + { + case LC_MAIN: + { + uint64_t entryPoint = reader.Read64(); + header.entryPoints.push_back({entryPoint, true}); + (void)reader.Read64(); // Stack start + break; + } + case LC_SEGMENT: // map the 32bit version to 64 bits + segment64.cmd = LC_SEGMENT_64; + reader.Read(&segment64.segname, 16); + segment64.vmaddr = reader.Read32(); + segment64.vmsize = reader.Read32(); + segment64.fileoff = reader.Read32(); + segment64.filesize = reader.Read32(); + segment64.maxprot = reader.Read32(); + segment64.initprot = reader.Read32(); + segment64.nsects = reader.Read32(); + segment64.flags = reader.Read32(); + if (strncmp(segment64.segname, "__TEXT\0", 7) == 0) + { + header.relocationBase = segment64.vmaddr; + header.textBase = segment64.vmaddr; + header.textBaseFileOffset = segment64.fileoff; + } + for (size_t j = 0; j < segment64.nsects; j++) + { + reader.Read(§.sectname, 16); + reader.Read(§.segname, 16); + sect.addr = reader.Read32(); + sect.size = reader.Read32(); + sect.offset = reader.Read32(); + sect.align = reader.Read32(); + sect.reloff = reader.Read32(); + sect.nreloc = reader.Read32(); + sect.flags = reader.Read32(); + sect.reserved1 = reader.Read32(); + sect.reserved2 = reader.Read32(); + // if the segment isn't mapped into virtual memory don't add the corresponding sections. + if (segment64.vmsize > 0) + { + header.sections.push_back(sect); + } + if (!strncmp(sect.sectname, "__mod_init_func", 15)) + header.moduleInitSections.push_back(sect); + if (!strncmp(sect.sectname, "__mod_term_func", 15)) + header.moduleTermSections.push_back(sect); + if ((sect.flags & (S_ATTR_SELF_MODIFYING_CODE | S_SYMBOL_STUBS)) + == (S_ATTR_SELF_MODIFYING_CODE | S_SYMBOL_STUBS)) + header.symbolStubSections.push_back(sect); + if ((sect.flags & S_NON_LAZY_SYMBOL_POINTERS) == S_NON_LAZY_SYMBOL_POINTERS) + header.symbolPointerSections.push_back(sect); + if ((sect.flags & S_LAZY_SYMBOL_POINTERS) == S_LAZY_SYMBOL_POINTERS) + header.symbolPointerSections.push_back(sect); + } + header.segments.push_back(segment64); + break; + case LC_SEGMENT_64: + segment64.cmd = LC_SEGMENT_64; + reader.Read(&segment64.segname, 16); + segment64.vmaddr = reader.Read64(); + segment64.vmsize = reader.Read64(); + segment64.fileoff = reader.Read64(); + segment64.filesize = reader.Read64(); + segment64.maxprot = reader.Read32(); + segment64.initprot = reader.Read32(); + segment64.nsects = reader.Read32(); + segment64.flags = reader.Read32(); + if (strncmp(segment64.segname, "__LINKEDIT", 10) == 0) + { + header.linkeditSegment = segment64; + header.linkeditPresent = true; + } + if (strncmp(segment64.segname, "__TEXT\0", 7) == 0) + { + header.relocationBase = segment64.vmaddr; + header.textBase = segment64.vmaddr; + header.textBaseFileOffset = segment64.fileoff; + } + for (size_t j = 0; j < segment64.nsects; j++) + { + reader.Read(§.sectname, 16); + reader.Read(§.segname, 16); + sect.addr = reader.Read64(); + sect.size = reader.Read64(); + sect.offset = reader.Read32(); + sect.align = reader.Read32(); + sect.reloff = reader.Read32(); + sect.nreloc = reader.Read32(); + sect.flags = reader.Read32(); + sect.reserved1 = reader.Read32(); + sect.reserved2 = reader.Read32(); + sect.reserved3 = reader.Read32(); + // if the segment isn't mapped into virtual memory don't add the corresponding sections. + if (segment64.vmsize > 0) + { + header.sections.push_back(sect); + } + + if (!strncmp(sect.sectname, "__mod_init_func", 15)) + header.moduleInitSections.push_back(sect); + if ((sect.flags & (S_ATTR_SELF_MODIFYING_CODE | S_SYMBOL_STUBS)) + == (S_ATTR_SELF_MODIFYING_CODE | S_SYMBOL_STUBS)) + header.symbolStubSections.push_back(sect); + if ((sect.flags & S_NON_LAZY_SYMBOL_POINTERS) == S_NON_LAZY_SYMBOL_POINTERS) + header.symbolPointerSections.push_back(sect); + if ((sect.flags & S_LAZY_SYMBOL_POINTERS) == S_LAZY_SYMBOL_POINTERS) + header.symbolPointerSections.push_back(sect); + } + header.segments.push_back(segment64); + break; + case LC_ROUTINES: // map the 32bit version to 64bits + header.routines64.cmd = LC_ROUTINES_64; + header.routines64.init_address = reader.Read32(); + header.routines64.init_module = reader.Read32(); + header.routines64.reserved1 = reader.Read32(); + header.routines64.reserved2 = reader.Read32(); + header.routines64.reserved3 = reader.Read32(); + header.routines64.reserved4 = reader.Read32(); + header.routines64.reserved5 = reader.Read32(); + header.routines64.reserved6 = reader.Read32(); + header.routinesPresent = true; + break; + case LC_ROUTINES_64: + header.routines64.cmd = LC_ROUTINES_64; + header.routines64.init_address = reader.Read64(); + header.routines64.init_module = reader.Read64(); + header.routines64.reserved1 = reader.Read64(); + header.routines64.reserved2 = reader.Read64(); + header.routines64.reserved3 = reader.Read64(); + header.routines64.reserved4 = reader.Read64(); + header.routines64.reserved5 = reader.Read64(); + header.routines64.reserved6 = reader.Read64(); + header.routinesPresent = true; + break; + case LC_FUNCTION_STARTS: + header.functionStarts.funcoff = reader.Read32(); + header.functionStarts.funcsize = reader.Read32(); + header.functionStartsPresent = true; + break; + case LC_SYMTAB: + header.symtab.symoff = reader.Read32(); + header.symtab.nsyms = reader.Read32(); + header.symtab.stroff = reader.Read32(); + header.symtab.strsize = reader.Read32(); + break; + case LC_DYSYMTAB: + header.dysymtab.ilocalsym = reader.Read32(); + header.dysymtab.nlocalsym = reader.Read32(); + header.dysymtab.iextdefsym = reader.Read32(); + header.dysymtab.nextdefsym = reader.Read32(); + header.dysymtab.iundefsym = reader.Read32(); + header.dysymtab.nundefsym = reader.Read32(); + header.dysymtab.tocoff = reader.Read32(); + header.dysymtab.ntoc = reader.Read32(); + header.dysymtab.modtaboff = reader.Read32(); + header.dysymtab.nmodtab = reader.Read32(); + header.dysymtab.extrefsymoff = reader.Read32(); + header.dysymtab.nextrefsyms = reader.Read32(); + header.dysymtab.indirectsymoff = reader.Read32(); + header.dysymtab.nindirectsyms = reader.Read32(); + header.dysymtab.extreloff = reader.Read32(); + header.dysymtab.nextrel = reader.Read32(); + header.dysymtab.locreloff = reader.Read32(); + header.dysymtab.nlocrel = reader.Read32(); + header.dysymPresent = true; + break; + case LC_DYLD_CHAINED_FIXUPS: + header.chainedFixups.dataoff = reader.Read32(); + header.chainedFixups.datasize = reader.Read32(); + header.chainedFixupsPresent = true; + break; + case LC_DYLD_INFO: + case LC_DYLD_INFO_ONLY: + header.dyldInfo.rebase_off = reader.Read32(); + header.dyldInfo.rebase_size = reader.Read32(); + header.dyldInfo.bind_off = reader.Read32(); + header.dyldInfo.bind_size = reader.Read32(); + header.dyldInfo.weak_bind_off = reader.Read32(); + header.dyldInfo.weak_bind_size = reader.Read32(); + header.dyldInfo.lazy_bind_off = reader.Read32(); + header.dyldInfo.lazy_bind_size = reader.Read32(); + header.dyldInfo.export_off = reader.Read32(); + header.dyldInfo.export_size = reader.Read32(); + header.exportTrie.dataoff = header.dyldInfo.export_off; + header.exportTrie.datasize = header.dyldInfo.export_size; + header.exportTriePresent = true; + header.dyldInfoPresent = true; + break; + case LC_DYLD_EXPORTS_TRIE: + header.exportTrie.dataoff = reader.Read32(); + header.exportTrie.datasize = reader.Read32(); + header.exportTriePresent = true; + break; + case LC_THREAD: + case LC_UNIXTHREAD: + /*while (reader.GetOffset() < nextOffset) + { + + thread_command thread; + thread.flavor = reader.Read32(); + thread.count = reader.Read32(); + switch (m_archId) + { + case MachOx64: + m_logger->LogDebug("x86_64 Thread state\n"); + if (thread.flavor != X86_THREAD_STATE64) + { + reader.SeekRelative(thread.count * sizeof(uint32_t)); + break; + } + //This wont be big endian so we can just read the whole thing + reader.Read(&thread.statex64, sizeof(thread.statex64)); + header.entryPoints.push_back({thread.statex64.rip, false}); + break; + case MachOx86: + m_logger->LogDebug("x86 Thread state\n"); + if (thread.flavor != X86_THREAD_STATE32) + { + reader.SeekRelative(thread.count * sizeof(uint32_t)); + break; + } + //This wont be big endian so we can just read the whole thing + reader.Read(&thread.statex86, sizeof(thread.statex86)); + header.entryPoints.push_back({thread.statex86.eip, false}); + break; + case MachOArm: + m_logger->LogDebug("Arm Thread state\n"); + if (thread.flavor != _ARM_THREAD_STATE) + { + reader.SeekRelative(thread.count * sizeof(uint32_t)); + break; + } + //This wont be big endian so we can just read the whole thing + reader.Read(&thread.statearmv7, sizeof(thread.statearmv7)); + header.entryPoints.push_back({thread.statearmv7.r15, false}); + break; + case MachOAarch64: + case MachOAarch6432: + m_logger->LogDebug("Aarch64 Thread state\n"); + if (thread.flavor != _ARM_THREAD_STATE64) + { + reader.SeekRelative(thread.count * sizeof(uint32_t)); + break; + } + reader.Read(&thread.stateaarch64, sizeof(thread.stateaarch64)); + header.entryPoints.push_back({thread.stateaarch64.pc, false}); + break; + case MachOPPC: + m_logger->LogDebug("PPC Thread state\n"); + if (thread.flavor != PPC_THREAD_STATE) + { + reader.SeekRelative(thread.count * sizeof(uint32_t)); + break; + } + //Read individual entries for endian reasons + header.entryPoints.push_back({reader.Read32(), false}); + (void)reader.Read32(); + (void)reader.Read32(); + //Read the rest of the structure + (void)reader.Read(&thread.stateppc.r1, sizeof(thread.stateppc) - (3 * 4)); + break; + case MachOPPC64: + m_logger->LogDebug("PPC64 Thread state\n"); + if (thread.flavor != PPC_THREAD_STATE64) + { + reader.SeekRelative(thread.count * sizeof(uint32_t)); + break; + } + header.entryPoints.push_back({reader.Read64(), false}); + (void)reader.Read64(); + (void)reader.Read64(); // Stack start + (void)reader.Read(&thread.stateppc64.r1, sizeof(thread.stateppc64) - (3 * 8)); + break; + default: + m_logger->LogError("Unknown archid: %x", m_archId); + } + + }*/ + break; + case LC_LOAD_DYLIB: + { + uint32_t offset = reader.Read32(); + if (offset < nextOffset) + { + reader.Seek(curOffset + offset); + std::string libname = reader.ReadCString(reader.GetOffset()); + header.dylibs.push_back(libname); + } + } + break; + case LC_BUILD_VERSION: + { + // m_logger->LogDebug("LC_BUILD_VERSION:"); + header.buildVersion.platform = reader.Read32(); + header.buildVersion.minos = reader.Read32(); + header.buildVersion.sdk = reader.Read32(); + header.buildVersion.ntools = reader.Read32(); + // m_logger->LogDebug("Platform: %s", BuildPlatformToString(header.buildVersion.platform).c_str()); + // m_logger->LogDebug("MinOS: %s", BuildToolVersionToString(header.buildVersion.minos).c_str()); + // m_logger->LogDebug("SDK: %s", BuildToolVersionToString(header.buildVersion.sdk).c_str()); + for (uint32_t j = 0; (i < header.buildVersion.ntools) && (j < 10); j++) + { + uint32_t tool = reader.Read32(); + uint32_t version = reader.Read32(); + header.buildToolVersions.push_back({tool, version}); + // m_logger->LogDebug("Build Tool: %s: %s", BuildToolToString(tool).c_str(), + // BuildToolVersionToString(version).c_str()); + } + break; + } + case LC_FILESET_ENTRY: + { + throw ReadException(); + } + default: + // m_logger->LogDebug("Unhandled command: %s : %" PRIu32 "\n", CommandToString(load.cmd).c_str(), + // load.cmdsize); + break; + } + if (reader.GetOffset() != nextOffset) + { + // m_logger->LogDebug("Didn't parse load command: %s fully %" PRIx64 ":%" PRIxPTR, + // CommandToString(load.cmd).c_str(), reader.GetOffset(), nextOffset); + } + reader.Seek(nextOffset); + } + + for (auto& section : header.sections) + { + char sectionName[17]; + memcpy(sectionName, section.sectname, sizeof(section.sectname)); + sectionName[16] = 0; + char segmentName[17]; + memcpy(segmentName, section.segname, sizeof(section.segname)); + segmentName[16] = 0; + if (header.identifierPrefix.empty()) + header.sectionNames.push_back(sectionName); + else + header.sectionNames.push_back(header.identifierPrefix + "::" + segmentName + ":" + sectionName); + } + } + catch (ReadException&) + { + return {}; + } + + return header; +} + +bool KernelCache::InitializeSegmentsForHeader(Ref<BinaryView> view, const KernelCacheMachOHeader& header, const KernelCacheImage& targetImage) +{ + // FIXME this uselessly loads chained fixups if an image is already loaded. + auto logger = LogRegistry::GetLogger("KernelCache", view->GetFile()->GetSessionId()); + + bool hasRegionsToLoad = false; + + auto reader = BinaryReader(view->GetParentView()); + + reader.Seek(0x10); + uint32_t ncmds = reader.Read32(); + uint64_t off = 0x20; // FIXME 32 bit + + uint64_t kernelBaseAddress = 0; + + uint32_t chainedFixupDataOff = 0; + uint32_t chainedFixupDataSize = 0; + + while (ncmds--) + { + reader.Seek(off); + uint32_t cmd = reader.Read32(); + uint32_t cmdsize = reader.Read32(); + if (cmd == LC_SEGMENT_64) + { + segment_command_64 segment {}; + reader.Read(&segment.segname, 16); + if (strncmp(segment.segname, "__TEXT\0", 7) == 0) + { + kernelBaseAddress = reader.Read64(); + } + } + if (cmd == LC_DYLD_CHAINED_FIXUPS) + { + chainedFixupDataOff = reader.Read32(); + chainedFixupDataSize = reader.Read32(); + } + off += cmdsize; + } + + BNRelocationInfo reloc; + memset(&reloc, 0, sizeof(BNRelocationInfo)); + reloc.type = StandardRelocationType; + reloc.size = 8; + reloc.nativeType = BINARYNINJA_MANUAL_RELOCATION; + logger->LogDebug("Processing Chained Fixups"); + + std::vector<std::pair<uint64_t, uint64_t>> relocations; + if (chainedFixupDataOff && chainedFixupDataSize) + { + BinaryReader parentReader(view->GetParentView()); + + try { + dyld_chained_fixups_header fixupsHeader {}; + uint64_t fixupHeaderAddress = chainedFixupDataOff; + parentReader.Seek(fixupHeaderAddress); + fixupsHeader.fixups_version = parentReader.Read32(); + fixupsHeader.starts_offset = parentReader.Read32(); + fixupsHeader.imports_offset = parentReader.Read32(); + fixupsHeader.symbols_offset = parentReader.Read32(); + fixupsHeader.imports_count = parentReader.Read32(); + fixupsHeader.imports_format = parentReader.Read32(); + fixupsHeader.symbols_format = parentReader.Read32(); + + logger->LogDebug("Chained Fixups: Header @ %llx // Fixups version %lx", fixupHeaderAddress, fixupsHeader.fixups_version); + + if (fixupsHeader.fixups_version > 0) + { + logger->LogError("Chained Fixup parsing failed. Unknown Fixups Version"); + throw ReadException(); + } + + uint64_t fixupStartsAddress = fixupHeaderAddress + fixupsHeader.starts_offset; + parentReader.Seek(fixupStartsAddress); + dyld_chained_starts_in_image segs {}; + segs.seg_count = parentReader.Read32(); + std::vector<uint32_t> segInfoOffsets {}; + for (size_t i = 0; i < segs.seg_count; i++) + { + segInfoOffsets.push_back(parentReader.Read32()); + } + for (auto offset : segInfoOffsets) + { + if (!offset) + continue; + + dyld_chained_starts_in_segment starts {}; + uint64_t startsAddr = fixupStartsAddress + offset; + parentReader.Seek(startsAddr); + starts.size = parentReader.Read32(); + starts.page_size = parentReader.Read16(); + starts.pointer_format = parentReader.Read16(); + starts.segment_offset = parentReader.Read64(); + starts.max_valid_pointer = parentReader.Read32(); + starts.page_count = parentReader.Read16(); + + uint8_t strideSize; + ChainedFixupPointerGeneric format; + + // Firmware formats will require digging up whatever place they're being used and reversing it. + // They are not handled by dyld. + switch (starts.pointer_format) { + case DYLD_CHAINED_PTR_ARM64E: + case DYLD_CHAINED_PTR_ARM64E_USERLAND: + case DYLD_CHAINED_PTR_ARM64E_USERLAND24: + strideSize = 8; + format = GenericArm64eFixupFormat; + break; + case DYLD_CHAINED_PTR_ARM64E_KERNEL: + strideSize = 4; + format = GenericArm64eFixupFormat; + break; + // case DYLD_CHAINED_PTR_ARM64E_FIRMWARE: Unsupported. + case DYLD_CHAINED_PTR_64: + case DYLD_CHAINED_PTR_64_OFFSET: + strideSize = 4; + format = Generic64FixupFormat; + break; + case DYLD_CHAINED_PTR_32: + case DYLD_CHAINED_PTR_32_CACHE: + strideSize = 4; + format = Generic32FixupFormat; + break; + case DYLD_CHAINED_PTR_32_FIRMWARE: + strideSize = 4; + format = Firmware32FixupFormat; + break; + case DYLD_CHAINED_PTR_64_KERNEL_CACHE: + strideSize = 4; + format = Kernel64Format; + break; + case DYLD_CHAINED_PTR_X86_64_KERNEL_CACHE: + strideSize = 1; + format = Kernel64Format; + break; + default: + { + logger->LogError("Chained Fixups: Unknown or unsupported pointer format %d, " + "unable to process chains for segment at @llx", starts.pointer_format, starts.segment_offset); + continue; + } + } + + uint16_t fmt = starts.pointer_format; + logger->LogDebug("Chained Fixups: Segment start @ %llx, fmt %d", starts.segment_offset, fmt); + + uint64_t pageStartsTableStartAddress = parentReader.GetOffset(); + std::vector<std::vector<uint16_t>> pageStartOffsets {}; + for (size_t i = 0; i < starts.page_count; i++) + { + // On armv7, Chained pointers here can have multiple starts. + // And if so, there's another table *overlapping* the table we're currently reading. + // dyld handles this through 'overflow indexing' + // This is technically supported on other archs however is not (currently) used. + parentReader.Seek(pageStartsTableStartAddress + (sizeof(uint16_t) * i)); + uint16_t start = parentReader.Read16(); + if ((start & DYLD_CHAINED_PTR_START_MULTI) && (start != DYLD_CHAINED_PTR_START_NONE)) + { + uint64_t overflowIndex = start & ~DYLD_CHAINED_PTR_START_MULTI; + std::vector<uint16_t> pageStartSubStarts; + parentReader.Seek(pageStartsTableStartAddress + (overflowIndex * sizeof(uint16_t))); + bool done = false; + while (!done) + { + uint16_t subPageStart = parentReader.Read16(); + if ((subPageStart & DYLD_CHAINED_PTR_START_LAST) == 0) + { + pageStartSubStarts.push_back(subPageStart); + } + else + { + pageStartSubStarts.push_back(subPageStart & ~DYLD_CHAINED_PTR_START_LAST); + done = true; + } + } + pageStartOffsets.push_back(pageStartSubStarts); + } + else + { + pageStartOffsets.push_back({start}); + } + } + + int i = -1; + for (auto pageStarts : pageStartOffsets) + { + i++; + uint64_t pageAddress = starts.segment_offset + (i * starts.page_size); + for (uint16_t start : pageStarts) + { + if (start == DYLD_CHAINED_PTR_START_NONE) + continue; + + uint64_t chainEntryAddress = pageAddress + start; + + bool fixupsDone = false; + + while (!fixupsDone) + { + ChainedFixupPointer pointer; + parentReader.Seek(chainEntryAddress); + if (format == Generic32FixupFormat || format == Firmware32FixupFormat) + pointer.raw32 = (uint32_t)(uintptr_t)parentReader.Read32(); + else + pointer.raw64 = (uintptr_t)parentReader.Read64(); + + bool bind = false; + uint64_t nextEntryStrideCount; + + switch (format) + { + case Generic32FixupFormat: + bind = pointer.generic32.bind.bind; + nextEntryStrideCount = pointer.generic32.rebase.next; + break; + case Generic64FixupFormat: + bind = pointer.generic64.bind.bind; + nextEntryStrideCount = pointer.generic64.rebase.next; + break; + case GenericArm64eFixupFormat: + bind = pointer.arm64e.bind.bind; + nextEntryStrideCount = pointer.arm64e.rebase.next; + break; + case Firmware32FixupFormat: + nextEntryStrideCount = pointer.firmware32.next; + bind = false; + break; + case Kernel64Format: + nextEntryStrideCount = pointer.kernel64.next; + bind = false; + } + + logger->LogTrace("Chained Fixups: @ 0x%llx ( 0x%llx ) - %d 0x%llx", chainEntryAddress, + kernelBaseAddress + (chainEntryAddress), + bind, nextEntryStrideCount); + + if (!bind) + { + uint64_t entryOffset; + switch (starts.pointer_format) + { + case DYLD_CHAINED_PTR_ARM64E: + case DYLD_CHAINED_PTR_ARM64E_KERNEL: + case DYLD_CHAINED_PTR_ARM64E_USERLAND: + case DYLD_CHAINED_PTR_ARM64E_USERLAND24: + { + if (pointer.arm64e.bind.auth) + entryOffset = pointer.arm64e.authRebase.target; + else + entryOffset = pointer.arm64e.rebase.target; + + if ( starts.pointer_format != DYLD_CHAINED_PTR_ARM64E || pointer.arm64e.bind.auth) + entryOffset += kernelBaseAddress; + + break; + } + case DYLD_CHAINED_PTR_64: + entryOffset = pointer.generic64.rebase.target; + break; + case DYLD_CHAINED_PTR_64_OFFSET: + entryOffset = pointer.generic64.rebase.target + kernelBaseAddress; + break; + // We expect only cases past this point will be applicable in this context. + case DYLD_CHAINED_PTR_64_KERNEL_CACHE: + case DYLD_CHAINED_PTR_X86_64_KERNEL_CACHE: + entryOffset = pointer.kernel64.target + kernelBaseAddress; + break; + case DYLD_CHAINED_PTR_32: + case DYLD_CHAINED_PTR_32_CACHE: + entryOffset = pointer.generic32.rebase.target; + break; + case DYLD_CHAINED_PTR_32_FIRMWARE: + entryOffset = pointer.firmware32.target; + break; + } + + // logger->LogInfo("Chained Fixups: Pointer at 0x%llx -> 0x%llx", view->GetStart() + chainEntryAddress, entryOffset); + + relocations.emplace_back(kernelBaseAddress + chainEntryAddress, entryOffset); + } + + chainEntryAddress += (nextEntryStrideCount * strideSize); + + if (chainEntryAddress > pageAddress + starts.page_size) + { + // Something is seriously wrong here. likely malformed binary, or our parsing failed elsewhere. + // This will log the pointer in mapped memory. + logger->LogError("Chained Fixups: Pointer at 0x%llx left page", + kernelBaseAddress + ((chainEntryAddress - (nextEntryStrideCount * strideSize)))); + fixupsDone = true; + } + + if (nextEntryStrideCount == 0) + fixupsDone = true; + } + } + } + } + } + catch (ReadException&) + { + logger->LogError("Chained Fixup parsing failed"); + } + } + + // Critical that we sort these as this optimization allows us to save a lot of time on image loading when we + // need to apply relocations later. Now is the best time, since we aren't going to find more relocs now. + std::sort(relocations.begin(), relocations.end(), + [](const std::pair<uint64_t, uint64_t>& a, const std::pair<uint64_t, uint64_t>& b) { + return a.first < b.first; + }); + + for (auto& region : targetImage.regions) + { + if (view->IsValidOffset(region.start)) + { + logger->LogDebug("Skipping region %s as it is already loaded.", region.prettyName.c_str()); + continue; + } + + hasRegionsToLoad = true; + + // Considerations at this point in processing: + // * View is already finalized + // * view (shown to user) has only one segment, at the kernel start address + // * All of our relevant data is contained in the Raw view. + view->AddAutoSegment(region.start, region.size, region.fileOffset, region.size, region.flags); + + auto begin = std::lower_bound(relocations.begin(), relocations.end(), region.start, + [](const std::pair<uint64_t, uint64_t>& reloc, uint64_t addr) { + return reloc.first < addr; + }); + + auto arch = view->GetDefaultArchitecture(); + // Process relocations until the VM address is beyond our region + for (auto it = begin; it != relocations.end() && it->first < region.start + region.size; ++it) { + reloc.address = it->first; + view->DefineRelocation(arch, reloc, it->second, reloc.address); + } + } + + if (!hasRegionsToLoad) + { + logger->LogWarn("No regions to load for image %s", header.installName.c_str()); + return false; + } + + return true; +} + +void KernelCache::InitializeHeader(Ref<BinaryView> view, KernelCacheMachOHeader header) +{ + Ref<Settings> settings = view->GetLoadSettings(KC_VIEW_NAME); + bool applyFunctionStarts = true; // FIXME + + view->AddAutoSection(header.identifierPrefix + "::__macho_header", header.textBase, header.ident.sizeofcmds + sizeof(mach_header_64), ReadOnlyDataSectionSemantics); + + for (size_t i = 0; i < header.sections.size(); i++) + { + if (!header.sections[i].size) + continue; + + std::string type; + BNSectionSemantics semantics = DefaultSectionSemantics; + switch (header.sections[i].flags & 0xff) + { + case S_REGULAR: + if (header.sections[i].flags & S_ATTR_PURE_INSTRUCTIONS) + { + type = "PURE_CODE"; + semantics = ReadOnlyCodeSectionSemantics; + } + else if (header.sections[i].flags & S_ATTR_SOME_INSTRUCTIONS) + { + type = "CODE"; + semantics = ReadOnlyCodeSectionSemantics; + } + else + { + type = "REGULAR"; + } + break; + case S_ZEROFILL: + type = "ZEROFILL"; + semantics = ReadWriteDataSectionSemantics; + break; + case S_CSTRING_LITERALS: + type = "CSTRING_LITERALS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_4BYTE_LITERALS: + type = "4BYTE_LITERALS"; + break; + case S_8BYTE_LITERALS: + type = "8BYTE_LITERALS"; + break; + case S_LITERAL_POINTERS: + type = "LITERAL_POINTERS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_NON_LAZY_SYMBOL_POINTERS: + type = "NON_LAZY_SYMBOL_POINTERS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_LAZY_SYMBOL_POINTERS: + type = "LAZY_SYMBOL_POINTERS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_SYMBOL_STUBS: + type = "SYMBOL_STUBS"; + semantics = ReadOnlyCodeSectionSemantics; + break; + case S_MOD_INIT_FUNC_POINTERS: + type = "MOD_INIT_FUNC_POINTERS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_MOD_TERM_FUNC_POINTERS: + type = "MOD_TERM_FUNC_POINTERS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_COALESCED: + type = "COALESCED"; + break; + case S_GB_ZEROFILL: + type = "GB_ZEROFILL"; + semantics = ReadWriteDataSectionSemantics; + break; + case S_INTERPOSING: + type = "INTERPOSING"; + break; + case S_16BYTE_LITERALS: + type = "16BYTE_LITERALS"; + break; + case S_DTRACE_DOF: + type = "DTRACE_DOF"; + break; + case S_LAZY_DYLIB_SYMBOL_POINTERS: + type = "LAZY_DYLIB_SYMBOL_POINTERS"; + semantics = ReadOnlyDataSectionSemantics; + break; + case S_THREAD_LOCAL_REGULAR: + type = "THREAD_LOCAL_REGULAR"; + break; + case S_THREAD_LOCAL_ZEROFILL: + type = "THREAD_LOCAL_ZEROFILL"; + break; + case S_THREAD_LOCAL_VARIABLES: + type = "THREAD_LOCAL_VARIABLES"; + break; + case S_THREAD_LOCAL_VARIABLE_POINTERS: + type = "THREAD_LOCAL_VARIABLE_POINTERS"; + break; + case S_THREAD_LOCAL_INIT_FUNCTION_POINTERS: + type = "THREAD_LOCAL_INIT_FUNCTION_POINTERS"; + break; + default: + type = "UNKNOWN"; + break; + } + if (i >= header.sectionNames.size()) + break; + if (strncmp(header.sections[i].sectname, "__text", sizeof(header.sections[i].sectname)) == 0) + semantics = ReadOnlyCodeSectionSemantics; + if (strncmp(header.sections[i].sectname, "__const", sizeof(header.sections[i].sectname)) == 0) + semantics = ReadOnlyDataSectionSemantics; + if (strncmp(header.sections[i].sectname, "__data", sizeof(header.sections[i].sectname)) == 0) + semantics = ReadWriteDataSectionSemantics; + if (strncmp(header.sections[i].segname, "__DATA_CONST", sizeof(header.sections[i].segname)) == 0) + semantics = ReadOnlyDataSectionSemantics; + + view->AddAutoSection(header.sectionNames[i], header.sections[i].addr, header.sections[i].size, semantics, + type, header.sections[i].align); + } + + auto typeLib = view->GetTypeLibrary(header.installName); + + BinaryReader virtualReader(view->GetParentView()); + + view->DefineDataVariable(header.textBase, Type::NamedType(view, QualifiedName("mach_header_64"))); + view->DefineAutoSymbol( + new Symbol(DataSymbol, "__macho_header::" + header.identifierPrefix, header.textBase, LocalBinding)); + + try + { + virtualReader.Seek(header.textBaseFileOffset + sizeof(mach_header_64)); + size_t sectionNum = 0; + uint64_t curVirtualOffset = header.textBase + sizeof(mach_header_64); + uint64_t curFileOffset = header.textBaseFileOffset + sizeof(mach_header_64); + for (size_t i = 0; i < header.ident.ncmds; i++) + { + load_command load; + curFileOffset = virtualReader.GetOffset(); + load.cmd = virtualReader.Read32(); + load.cmdsize = virtualReader.Read32(); + uint64_t nextOffset = curFileOffset + load.cmdsize; + switch (load.cmd) + { + case LC_SEGMENT: + { + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("segment_command"))); + virtualReader.SeekRelative(5 * 8); + size_t numSections = virtualReader.Read32(); + virtualReader.SeekRelative(4); + for (size_t j = 0; j < numSections; j++) + { + view->DefineDataVariable( + virtualReader.GetOffset(), Type::NamedType(view, QualifiedName("section"))); + view->DefineAutoSymbol(new Symbol(DataSymbol, + "__macho_section::" + header.identifierPrefix + "_[" + std::to_string(sectionNum++) + "]", + virtualReader.GetOffset(), LocalBinding)); + virtualReader.SeekRelative((8 * 8) + 4); + } + break; + } + case LC_SEGMENT_64: + { + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("segment_command_64"))); + virtualReader.SeekRelative(7 * 8); + size_t numSections = virtualReader.Read32(); + virtualReader.SeekRelative(4); + for (size_t j = 0; j < numSections; j++) + { + view->DefineDataVariable( + virtualReader.GetOffset(), Type::NamedType(view, QualifiedName("section_64"))); + view->DefineAutoSymbol(new Symbol(DataSymbol, + "__macho_section_64::" + header.identifierPrefix + "_[" + std::to_string(sectionNum++) + "]", + virtualReader.GetOffset(), LocalBinding)); + virtualReader.SeekRelative(10 * 8); + } + break; + } + case LC_SYMTAB: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("symtab"))); + break; + case LC_DYSYMTAB: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("dysymtab"))); + break; + case LC_UUID: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("uuid"))); + break; + case LC_ID_DYLIB: + case LC_LOAD_DYLIB: + case LC_REEXPORT_DYLIB: + case LC_LOAD_WEAK_DYLIB: + case LC_LOAD_UPWARD_DYLIB: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("dylib_command"))); + if (load.cmdsize - 24 <= 150) + view->DefineDataVariable( + curVirtualOffset + 24, Type::ArrayType(Type::IntegerType(1, true), load.cmdsize - 24)); + break; + case LC_CODE_SIGNATURE: + case LC_SEGMENT_SPLIT_INFO: + case LC_FUNCTION_STARTS: + case LC_DATA_IN_CODE: + case LC_DYLIB_CODE_SIGN_DRS: + case LC_DYLD_EXPORTS_TRIE: + case LC_DYLD_CHAINED_FIXUPS: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("linkedit_data"))); + break; + case LC_ENCRYPTION_INFO: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("encryption_info"))); + break; + case LC_VERSION_MIN_MACOSX: + case LC_VERSION_MIN_IPHONEOS: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("version_min"))); + break; + case LC_DYLD_INFO: + case LC_DYLD_INFO_ONLY: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("dyld_info"))); + break; + default: + view->DefineDataVariable(curVirtualOffset, Type::NamedType(view, QualifiedName("load_command"))); + break; + } + + view->DefineAutoSymbol(new Symbol(DataSymbol, + "__macho_load_command::" + header.identifierPrefix + "_[" + std::to_string(i) + "]", curVirtualOffset, + LocalBinding)); + curVirtualOffset += (nextOffset - curFileOffset); + virtualReader.Seek(nextOffset); + } + } + catch (ReadException&) + { + LogError("Error when applying Mach-O header types at %" PRIx64, header.textBase); + } + + if (applyFunctionStarts && header.functionStartsPresent && header.linkeditPresent) + { + size_t i = 0; + uint64_t curfunc = header.textBase; + uint64_t curOffset; + + auto funcStarts = view->GetParentView()->ReadBuffer(header.functionStarts.funcoff, header.functionStarts.funcsize); + + while (i < header.functionStarts.funcsize) + { + curOffset = readLEB128(funcStarts, header.functionStarts.funcsize, i); + curfunc += curOffset; + // LogError("0x%llx, 0x%llx", header.textBase, curOffset); + if (curOffset == 0) + continue; + uint64_t target = curfunc; + Ref<Platform> targetPlatform = view->GetDefaultPlatform(); + view->AddFunctionForAnalysis(targetPlatform, target); + } + } + + view->BeginBulkModifySymbols(); + ParseSymbolTable(view, header); + + BinaryReader reader(view); + + size_t modInitFuncCnt = 0; + for (const auto& moduleInitSection : header.moduleInitSections) + { + // The mod_init section contains a list of function pointers called at initialization + // if we don't have a defined entrypoint then use the first one in the list as the entrypoint + size_t i = 0; + reader.Seek(moduleInitSection.addr); + for (; i < (moduleInitSection.size / view->GetAddressSize()); i++) + { + uint64_t target = (view->GetAddressSize() == 4) ? reader.Read32() : reader.Read64(); + if (!view->IsValidOffset(target)) + continue; + Ref<Platform> targetPlatform = view->GetDefaultPlatform()->GetAssociatedPlatformByAddress(target); + auto name = header.identifierPrefix + "_init_func_" + std::to_string(modInitFuncCnt++); + view->AddEntryPointForAnalysis(targetPlatform, target); + auto symbol = new Symbol(FunctionSymbol, name, target, GlobalBinding); + view->DefineAutoSymbol(symbol); + } + } + + view->EndBulkModifySymbols(); +} + + +std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> KernelCache::ParseSymbolTable(Ref<BinaryView> view, KernelCacheMachOHeader header, bool defineSymbolsInView) +{ + if (header.symtab.symoff != 0 && header.linkeditPresent) + { + // Mach-O View symtab processing with + // a ton of stuff cut out so it can work + // auto symtab = reader->ReadBuffer(header.symtab.symoff, header.symtab.nsyms * sizeof(nlist_64)); + auto strtab = view->GetParentView()->ReadBuffer(header.symtab.stroff, header.symtab.strsize); + nlist_64 sym; + memset(&sym, 0, sizeof(sym)); + std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> symbolInfos; + for (size_t i = 0; i < header.symtab.nsyms; i++) + { + view->GetParentView()->Read(&sym, header.symtab.symoff + i * sizeof(nlist_64), sizeof(nlist_64)); + if (sym.n_strx >= header.symtab.strsize || ((sym.n_type & N_TYPE) == N_INDR)) + continue; + + std::string symbol((char*)strtab.GetDataAt(sym.n_strx)); + // BNLogError("%s: 0x%llx", symbol.c_str(), sym.n_value); + if (symbol == "<redacted>") + continue; + + BNSymbolType type = DataSymbol; + uint32_t flags; + if ((sym.n_type & N_TYPE) == N_SECT && sym.n_sect > 0 && (size_t)(sym.n_sect - 1) < header.sections.size()) + {} + else if ((sym.n_type & N_TYPE) == N_ABS) + {} + else if ((sym.n_type & 0x1)) + { + type = ExternalSymbol; + } + else + continue; + + for (auto s : header.sections) + { + if (s.addr < sym.n_value) + { + if (s.addr + s.size > sym.n_value) + { + flags = s.flags; + } + } + } + + if (type != ExternalSymbol) + { + if ((flags & S_ATTR_PURE_INSTRUCTIONS) == S_ATTR_PURE_INSTRUCTIONS + || (flags & S_ATTR_SOME_INSTRUCTIONS) == S_ATTR_SOME_INSTRUCTIONS) + type = FunctionSymbol; + else + type = DataSymbol; + } + if ((sym.n_desc & N_ARM_THUMB_DEF) == N_ARM_THUMB_DEF) + sym.n_value++; + + std::string rawName = symbol; + std::string shortName = symbol; + std::string fullName = symbol; + Ref<Type> typeRef = nullptr; + + if (view->GetDefaultArchitecture()) + { + QualifiedName demangledName; + Ref<Type> demangledType; + bool simplify = Settings::Instance()->Get<bool>("analysis.types.templateSimplifier", view); + if (DemangleGeneric(view->GetDefaultArchitecture(), rawName, demangledType, demangledName, view, simplify)) + { + shortName = demangledName.GetString(); + fullName = shortName; + if (demangledType) + fullName += demangledType->GetStringAfterName(); + if (!typeRef && !view->GetDefaultPlatform()->GetFunctionByName(rawName)) + typeRef = demangledType; + } + else + { + LogDebug("Failed to demangle name: '%s'\n", rawName.c_str()); + } + } + + if (defineSymbolsInView) + { + auto symbolObj = new Symbol(type, shortName, fullName, rawName, sym.n_value, GlobalBinding); + + if (typeRef) + view->DefineAutoSymbolAndVariableOrFunction(view->GetDefaultPlatform(), symbolObj, typeRef); + else + { + view->DefineAutoSymbol(symbolObj); + if (type == FunctionSymbol) + { + Ref<Platform> targetPlatform = view->GetDefaultPlatform(); + view->AddFunctionForAnalysis(targetPlatform, sym.n_value); + } + } + } + symbolInfos.push_back({sym.n_value, {type, fullName}}); + } + return symbolInfos; + } + return {}; +} + + +struct ExportNode +{ + std::string text; + uint64_t offset; + uint64_t flags; +}; + + +void KernelCache::ReadExportNode(Ref<BinaryView> view, std::vector<Ref<Symbol>>& symbolList, KernelCacheMachOHeader& header, DataBuffer& buffer, uint64_t textBase, + const std::string& currentText, size_t cursor, uint32_t endGuard) +{ + + if (cursor > endGuard) + throw ReadException(); + + uint64_t terminalSize = readValidULEB128(buffer, cursor); + uint64_t childOffset = cursor + terminalSize; + if (terminalSize != 0) { + uint64_t imageOffset = 0; + uint64_t flags = readValidULEB128(buffer, cursor); + if (!(flags & EXPORT_SYMBOL_FLAGS_REEXPORT)) + { + imageOffset = readValidULEB128(buffer, cursor); + // auto symbolType = view->GetAnalysisFunctionsForAddress(textBase + imageOffset).size() ? FunctionSymbol : DataSymbol; + { + if (!currentText.empty() && textBase + imageOffset) + { + uint32_t flags; + BNSymbolType type; + for (auto s : header.sections) + { + if (s.addr < textBase + imageOffset) + { + if (s.addr + s.size > textBase + imageOffset) + { + flags = s.flags; + } + } + } + if ((flags & S_ATTR_PURE_INSTRUCTIONS) == S_ATTR_PURE_INSTRUCTIONS + || (flags & S_ATTR_SOME_INSTRUCTIONS) == S_ATTR_SOME_INSTRUCTIONS) + type = FunctionSymbol; + else + type = DataSymbol; + +#if EXPORT_TRIE_DEBUG + // BNLogInfo("export: %s -> 0x%llx", n.text.c_str(), image.baseAddress + n.offset); +#endif + auto sym = new Symbol(type, currentText, textBase + imageOffset); + symbolList.push_back(sym); + } + } + } + } + cursor = childOffset; + uint8_t childCount = buffer[cursor]; + cursor++; + if (cursor > endGuard) + throw ReadException(); + for (uint8_t i = 0; i < childCount; ++i) + { + std::string childText; + while (buffer[cursor] != 0 & cursor <= endGuard) + childText.push_back(buffer[cursor++]); + cursor++; + if (cursor > endGuard) + throw ReadException(); + auto next = readValidULEB128(buffer, cursor); + if (next == 0) + throw ReadException(); + ReadExportNode(view, symbolList, header, buffer, textBase, currentText + childText, next, endGuard); + } +} + + +std::vector<Ref<Symbol>> KernelCache::ParseExportTrie(Ref<BinaryView> view, KernelCacheMachOHeader header) +{ + if (!header.exportTriePresent || !header.exportTrie.dataoff || !header.exportTrie.datasize) + return {}; + + std::vector<Ref<Symbol>> symbols; + try + { + std::vector<ExportNode> nodes; + + DataBuffer buffer = view->GetParentView()->ReadBuffer(header.exportTrie.dataoff, header.exportTrie.datasize); + ReadExportNode(view, symbols, header, buffer, header.textBase, "", 0, header.exportTrie.datasize); + } + catch (std::exception& e) + { + BNLogError("Failed to load Export Trie"); + } + return symbols; +} + +std::vector<std::string> KernelCache::GetAvailableImages() +{ + std::vector<std::string> installNames; + for (const auto& img : m_cacheInfo->images) + { + installNames.push_back(img.installName); + } + return installNames; +} + + +std::vector<KernelCacheImage> KernelCache::GetLoadedImages() +{ + std::lock_guard lock(m_viewSpecificState->stateMutex); + std::vector<KernelCacheImage> images; + for (const auto& [fileStart, image] : m_viewSpecificState->state.loadedImages) + { + images.push_back(image); + } + return images; +} + + +std::vector<std::pair<uint64_t, std::pair<std::string, std::string>>> KernelCache::LoadAllSymbolsAndWait() +{ + std::vector<std::pair<uint64_t, std::pair<std::string, std::string>>> symbols; + for (const auto& img : m_cacheInfo->images) + { + auto header = HeaderForFileAddress(img.headerFileLocation); + if (header) + { + auto newSymbolList = ParseSymbolTable(m_kcView, *header, false); + for (const auto& symbol : newSymbolList) + { + if (symbol.first != 0) + symbols.push_back({symbol.first, {header->installName, symbol.second.second}}); + } + } + } + return symbols; +} + + +std::string KernelCache::SerializedImageHeaderForVMAddress(uint64_t address) +{ + auto header = HeaderForVMAddress(address); + if (header) + { + return header->AsString(); + } + return ""; +} + + +std::string KernelCache::SerializedImageHeaderForName(std::string name) +{ + if (auto it = m_cacheInfo->imageStarts.find(name); it != m_cacheInfo->imageStarts.end()) + { + if (auto header = HeaderForFileAddress(it->second)) + { + return header->AsString(); + } + } + return ""; +} + + + +bool KernelCache::SaveCacheInfoToKCView(std::lock_guard<std::mutex>&) +{ + if (!m_cacheInfo) + return false; + + // The initial load should only populate `m_cacheInfo` and should not modify any state. + assert(m_modifiedState->loadedImages.size() == 0); + + auto data = m_cacheInfo->AsMetadata(); + m_kcView->StoreMetadata(KernelCacheMetadata::Tag, data); + m_kcView->GetParentView()->StoreMetadata(KernelCacheMetadata::Tag, data); + + { + std::lock_guard lock(m_viewSpecificState->cacheInfoMutex); + if (m_cacheInfo && !m_viewSpecificState->cacheInfo) + m_viewSpecificState->cacheInfo = m_cacheInfo; + else if (m_cacheInfo != m_viewSpecificState->cacheInfo) + abort(); + } + + m_metadataValid = true; + return true; +} + + +bool KernelCache::SaveModifiedStateToKCView(std::lock_guard<std::mutex>&) +{ + if (!m_kcView) + return false; + + { + std::lock_guard lock(m_viewSpecificState->stateMutex); + + uint64_t modificationNumber = m_viewSpecificState->savedModifications++; + if (modificationNumber == 0) + { + // The cached state in the view-specific state has not yet been saved. + // For the initial load of a shared cache this will be empty, but if + // the shared cache has been loaded from a database then this will + // contain the full state that was saved. + std::string metadataKey = KernelCacheMetadata::ModifiedStateTagPrefix + std::to_string(modificationNumber); + auto data = m_viewSpecificState->state.AsMetadata(m_viewSpecificState->viewState); + + m_kcView->StoreMetadata(metadataKey, data); + m_kcView->GetParentView()->StoreMetadata(metadataKey, data); + modificationNumber = m_viewSpecificState->savedModifications++; + } + + std::string metadataKey = KernelCacheMetadata::ModifiedStateTagPrefix + std::to_string(modificationNumber); + auto data = m_modifiedState->AsMetadata(); + + m_kcView->StoreMetadata(metadataKey, data); + m_kcView->GetParentView()->StoreMetadata(metadataKey, data); + + Ref<Metadata> count = new Metadata(m_viewSpecificState->savedModifications); + m_kcView->StoreMetadata(KernelCacheMetadata::ModifiedStateCountTag, count); + m_kcView->GetParentView()->StoreMetadata(KernelCacheMetadata::ModifiedStateCountTag, count); + + m_viewSpecificState->state.loadedImages.merge(m_modifiedState->loadedImages); + // `merge` will move a node to the target map if the corresponding key does not yet exist. + // If we've redundantly loaded images, we may be left with symbols in the source maps. + m_modifiedState->loadedImages.clear(); + + // Clean up any metadata entries past the current modification number. + // These can happen after being loaded from a database as all modifications are + // merged into a single state object and the modification count is reset to zero. + for (size_t i = modificationNumber + 1; i < std::numeric_limits<size_t>::max(); ++i) + { + std::string metadataKey = KernelCacheMetadata::ModifiedStateTagPrefix + std::to_string(i); + bool done = true; + if (m_kcView->QueryMetadata(metadataKey)) + { + done = false; + m_kcView->RemoveMetadata(metadataKey); + } + if (m_kcView->GetParentView()->QueryMetadata(metadataKey)) + { + done = false; + m_kcView->GetParentView()->RemoveMetadata(metadataKey); + } + if (done) + break; + } + } + + if (m_modifiedState->viewState) + { + m_viewSpecificState->viewState = m_modifiedState->viewState.value(); + m_modifiedState->viewState = std::nullopt; + } + + m_metadataValid = true; + + return true; +} + + +const std::string KernelCacheMetadata::Tag = "KERNELCACHE-KernelCacheData"; +const std::string KernelCacheMetadata::CacheInfoTag = "KERNELCACHE-CacheInfo"; +const std::string KernelCacheMetadata::ModifiedStateTagPrefix = "KERNELCACHE-ModifiedState-"; +const std::string KernelCacheMetadata::ModifiedStateCountTag = "KERNELCACHE-ModifiedState-Count"; + +KernelCacheMetadata::~KernelCacheMetadata() = default; +KernelCacheMetadata::KernelCacheMetadata(KernelCacheMetadata&&) = default; +KernelCacheMetadata& KernelCacheMetadata::operator=(KernelCacheMetadata&&) = default; + +KernelCacheMetadata::KernelCacheMetadata(KernelCache::CacheInfo cacheInfo, KernelCache::ModifiedState state) : + cacheInfo(std::make_unique<KernelCache::CacheInfo>(std::move(cacheInfo))), + state(std::make_unique<KernelCache::ModifiedState>(std::move(state))) +{} + + +// static +bool KernelCacheMetadata::ViewHasMetadata(BinaryView* view) +{ + return view->QueryMetadata(Tag); +} + +// static +std::optional<KernelCacheMetadata> KernelCacheMetadata::LoadFromView(BinaryView* view) +{ + Ref<Metadata> viewMetadata = view->QueryMetadata(Tag); + if (!viewMetadata) + return std::nullopt; + + auto cacheInfo = KernelCache::CacheInfo::LoadFromString(viewMetadata->GetString()); + if (!cacheInfo) + return std::nullopt; + + auto modifiedState = KernelCache::ModifiedState::LoadAll(view, *cacheInfo); + return KernelCacheMetadata(std::move(*cacheInfo), std::move(modifiedState)); +} + +std::string KernelCacheMetadata::InstallNameForImageBaseAddress(uint64_t baseAddress) const +{ + auto it = std::find_if(cacheInfo->imageStarts.begin(), cacheInfo->imageStarts.end(), [=](auto& pair) { + return pair.second == baseAddress; + }); + + if (it == cacheInfo->imageStarts.end()) + return ""; + + return it->first; +} + +std::vector<KernelCacheImage> KernelCacheMetadata::LoadedImages() +{ + std::vector<KernelCacheImage> images; + for (const auto& image : state->loadedImages) + { + images.push_back(image.second); + } + return images; +} + +} + +extern "C" +{ + BNKernelCache* BNGetKernelCache(BNBinaryView* data) + { + if (!data) + return nullptr; + + Ref<BinaryView> view = new BinaryView(BNNewViewReference(data)); + if (auto cache = KernelCache::GetFromKCView(view)) + { + cache->AddAPIRef(); + return cache->GetAPIObject(); + } + + return nullptr; + } + + BNKernelCache* BNNewKernelCacheReference(BNKernelCache* cache) + { + if (!cache->object) + return nullptr; + + cache->object->AddAPIRef(); + return cache; + } + + void BNFreeKernelCacheReference(BNKernelCache* cache) + { + if (!cache->object) + return; + + cache->object->ReleaseAPIRef(); + } + + bool BNKCViewLoadImageWithInstallName(BNKernelCache* cache, char* name) + { + std::string imageName = std::string(name); + // FIXME !!!!!!!! BNFreeString(name); + + if (cache->object) + return cache->object->LoadImageWithInstallName(imageName); + + return false; + } + + bool BNKCViewLoadImageContainingAddress(BNKernelCache* cache, uint64_t address) + { + if (cache->object) + { + return cache->object->LoadImageContainingAddress(address); + } + + return false; + } + + char** BNKCViewGetInstallNames(BNKernelCache* cache, size_t* count) + { + if (cache->object) + { + auto value = cache->object->GetAvailableImages(); + *count = value.size(); + + std::vector<const char*> cstrings; + for (size_t i = 0; i < value.size(); i++) + { + cstrings.push_back(value[i].c_str()); + } + return BNAllocStringList(cstrings.data(), cstrings.size()); + } + *count = 0; + return nullptr; + } + + BNKCSymbolRep* BNKCViewLoadAllSymbolsAndWait(BNKernelCache* cache, size_t* count) + { + if (cache->object) + { + auto value = cache->object->LoadAllSymbolsAndWait(); + *count = value.size(); + + BNKCSymbolRep* symbols = (BNKCSymbolRep*)malloc(sizeof(BNKCSymbolRep) * value.size()); + for (size_t i = 0; i < value.size(); i++) + { + symbols[i].address = value[i].first; + symbols[i].name = BNAllocString(value[i].second.second.c_str()); + symbols[i].image = BNAllocString(value[i].second.first.c_str()); + } + return symbols; + } + *count = 0; + return nullptr; + } + + void BNKCViewFreeSymbols(BNKCSymbolRep* symbols, size_t count) + { + for (size_t i = 0; i < count; i++) + { + BNFreeString(symbols[i].name); + BNFreeString(symbols[i].image); + } + delete symbols; + } + + char* BNKCViewGetNameForAddress(BNKernelCache* cache, uint64_t address) + { + if (cache->object) + { + return BNAllocString(cache->object->NameForAddress(address).c_str()); + } + + return nullptr; + } + + char* BNKCViewGetImageNameForAddress(BNKernelCache* cache, uint64_t address) + { + if (cache->object) + { + return BNAllocString(cache->object->ImageNameForAddress(address).c_str()); + } + + return nullptr; + } + + uint64_t BNKCViewLoadedImageCount(BNKernelCache* cache) + { + // FIXME? + return 0; + } + + BNKCViewState BNKCViewGetState(BNKernelCache* cache) + { + if (cache->object) + { + return (BNKCViewState)cache->object->ViewState(); + } + + return BNKCViewState::Unloaded; + } + + void BNKCViewFreeLoadedRegions(BNKCMappedMemoryRegion* images, size_t count) + { + for (size_t i = 0; i < count; i++) + { + BNFreeString(images[i].name); + } + delete images; + } + + BNKCImage* BNKCViewGetAllImages(BNKernelCache* cache, size_t* count) + { + if (cache->object) + { + auto viewImageHeaders = cache->object->AllImageHeaders(); + *count = viewImageHeaders.size(); + BNKCImage* images = (BNKCImage*)malloc(sizeof(BNKCImage) * viewImageHeaders.size()); + size_t i = 0; + for (const auto& [baseAddress, header] : viewImageHeaders) + { + images[i].name = BNAllocString(header.installName.c_str()); + images[i].headerFileAddress = baseAddress; + images[i].mappingCount = header.sections.size(); + images[i].mappings = (BNKCImageMemoryMapping*)malloc(sizeof(BNKCImageMemoryMapping) * header.sections.size()); + for (size_t j = 0; j < header.sections.size(); j++) + { + images[i].mappings[j].rawViewOffset = header.sections[j].offset; + images[i].mappings[j].vmAddress = header.sections[j].addr; + images[i].mappings[j].size = header.sections[j].size; + images[i].mappings[j].name = BNAllocString(header.sectionNames[j].c_str()); + } + i++; + } + return images; + } + *count = 0; + return nullptr; + } + + void BNKCViewFreeAllImages(BNKCImage* images, size_t count) + { + for (size_t i = 0; i < count; i++) + { + for (size_t j = 0; j < images[i].mappingCount; j++) + { + BNFreeString(images[i].mappings[j].name); + } + delete[] images[i].mappings; + BNFreeString(images[i].name); + } + delete[] images; + } + + char* BNKCViewGetImageHeaderForAddress(BNKernelCache* cache, uint64_t address) + { + if (cache->object) + { + auto header = cache->object->SerializedImageHeaderForVMAddress(address); + return BNAllocString(header.c_str()); + } + + return nullptr; + } + + char* BNKCViewGetImageHeaderForName(BNKernelCache* cache, char* name) + { + std::string imageName = std::string(name); + BNFreeString(name); + if (cache->object) + { + auto header = cache->object->SerializedImageHeaderForName(imageName); + return BNAllocString(header.c_str()); + } + + return nullptr; + } + + BNKCViewLoadProgress BNKCViewGetLoadProgress(uint64_t sessionID) + { + if (auto viewSpecificState = ViewSpecificStateForId(sessionID, false)) + return viewSpecificState->progress; + + return LoadProgressNotStarted; + } + + uint64_t BNKCViewFastGetImageCount(BNBinaryView* data) + { + Ref<BinaryView> view = new BinaryView(BNNewViewReference(data)); + if (view) + return KernelCache::FastGetImageCount(view); + return 0; + } + + BNKCImage* BNKCViewGetLoadedImages(BNKernelCache* cache, size_t* count) + { + if (cache->object) + { + auto images = cache->object->GetLoadedImages(); + *count = images.size(); + BNKCImage* bnImages = new BNKCImage[images.size()]; + for (size_t i = 0; i < images.size(); i++) + { + bnImages[i].name = BNAllocString(images[i].installName.c_str()); + bnImages[i].headerFileAddress = images[i].headerFileLocation; + bnImages[i].mappingCount = images[i].regions.size(); + bnImages[i].mappings = new BNKCImageMemoryMapping[images[i].regions.size()]; + for (size_t j = 0; j < images[i].regions.size(); j++) + { + bnImages[i].mappings[j].rawViewOffset = images[i].regions[j].fileOffset; + bnImages[i].mappings[j].vmAddress = images[i].regions[j].start; + bnImages[i].mappings[j].size = images[i].regions[j].size; + bnImages[i].mappings[j].name = BNAllocString(images[i].regions[j].prettyName.c_str()); + } + } + return bnImages; + } + *count = 0; + return nullptr; + } + + void BNKCViewFreeLoadedImages(BNKCImage* images, size_t count) + { + for (size_t i = 0; i < count; i++) + { + for (size_t j = 0; j < images[i].mappingCount; j++) + { + BNFreeString(images[i].mappings[j].name); + } + delete[] images[i].mappings; + BNFreeString(images[i].name); + } + delete[] images; + } +} +#ifdef __cplusplus +extern "C" { +#endif + + void RegisterTransformers(); + +#ifdef __cplusplus +} +#endif + +void InitKernelcache() +{ + InitKCViewType(); + RegisterTransformers(); +} + +namespace KernelCacheCore { + +void Deserialize( + DeserializationContext& context, std::string_view name, std::optional<std::pair<uint64_t, uint64_t>>& value) +{ + if (!context.doc.HasMember(name.data())) + { + value = std::nullopt; + return; + } + + auto array = context.doc[name.data()].GetArray(); + value = {array[0].GetUint64(), array[1].GetUint64()}; +} + +void Serialize(SerializationContext& context, const Ref<Symbol>& value) +{ + context.writer.StartArray(); + Serialize(context, value->GetRawNameRef()); + Serialize(context, value->GetAddress()); + Serialize(context, value->GetType()); + context.writer.EndArray(); +} + +void Serialize(SerializationContext& context, const std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>>& value) +{ + context.writer.StartArray(); + for (const auto& [_, symbol] : *value) + { + Serialize(context, symbol); + } + context.writer.EndArray(); +} + +void Serialize(SerializationContext& context, const std::shared_ptr<std::vector<Ref<Symbol>>>& value) +{ + Serialize(context, *value); +} + +void Deserialize(DeserializationContext& context, std::string_view name, + std::unordered_map<uint64_t, std::shared_ptr<std::unordered_map<uint64_t, Ref<Symbol>>>>& value) +{ + auto array = context.doc[name.data()].GetArray(); + for (auto& pair : array) + { + auto symbols_array = pair[1].GetArray(); + std::unordered_map<uint64_t, Ref<Symbol>> symbols; + for (auto& symbol_value : symbols_array) + { + auto symbol_array = symbol_value.GetArray(); + std::string symbolName = symbol_array[0].GetString(); + uint64_t address = symbol_array[1].GetUint64(); + BNSymbolType type = (BNSymbolType)symbol_array[2].GetUint(); + symbols.insert({address, new Symbol(type, symbolName, address)}); + } + value[pair[0].GetUint64()] = std::make_shared<std::unordered_map<uint64_t, Ref<Symbol>>>(std::move(symbols)); + } +} + +void Deserialize(DeserializationContext& context, std::string_view name, + std::unordered_map<uint64_t, std::shared_ptr<std::vector<Ref<Symbol>>>>& value) +{ + auto array = context.doc[name.data()].GetArray(); + for (auto& pair : array) + { + auto symbols_array = pair[1].GetArray(); + std::vector<Ref<Symbol>> symbols; + symbols.reserve(symbols_array.Size()); + for (auto& symbol_value : symbols_array) + { + auto symbol_array = symbol_value.GetArray(); + std::string symbolName = symbol_array[0].GetString(); + uint64_t address = symbol_array[1].GetUint64(); + BNSymbolType type = (BNSymbolType)symbol_array[2].GetUint(); + symbols.push_back(new Symbol(type, symbolName, address)); + } + value[pair[0].GetUint64()] = std::make_shared<std::vector<Ref<Symbol>>>(std::move(symbols)); + } +} + +void Deserialize(DeserializationContext& context, std::string_view name, std::optional<KCViewState>& viewState) +{ + auto& value = context.doc[name.data()]; + if (value.IsNull()) + viewState = std::nullopt; + else + viewState = (KCViewState)value.GetUint(); +} + + +void Serialize(SerializationContext& context, const std::vector<MemoryRegion>& value) +{ + context.writer.StartArray(); + for (const auto& region : value) + { + context.writer.StartArray(); + Serialize(context, region.prettyName); + Serialize(context, region.start); + Serialize(context, region.size); + Serialize(context, region.fileOffset); + Serialize(context, (uint64_t)region.flags); + Serialize(context, (uint8_t)region.type); + context.writer.EndArray(); + } + context.writer.EndArray(); +} + +void Deserialize(DeserializationContext& context, std::string_view name, std::vector<MemoryRegion>& value) +{ + auto array = context.doc[name.data()].GetArray(); + value.reserve(array.Size()); + for (auto& region : array) + { + auto region_array = region.GetArray(); + MemoryRegion newRegion; + newRegion.prettyName = region_array[0].GetString(); + newRegion.start = region_array[1].GetUint64(); + newRegion.size = region_array[2].GetUint64(); + newRegion.fileOffset = region_array[3].GetUint64(); + newRegion.flags = (BNSegmentFlag)region_array[4].GetUint64(); + newRegion.type = (MemoryRegion::Type)region_array[5].GetUint(); + value.push_back(newRegion); + } +} + +void Serialize(SerializationContext& context, const std::vector<KernelCacheImage>& value) +{ + context.writer.StartArray(); + for (const auto& image : value) + { + Serialize(context, image); + } + context.writer.EndArray(); +} + + +void Deserialize(DeserializationContext& context, std::string_view name, std::vector<KernelCacheImage>& value) +{ + auto array = context.doc[name.data()].GetArray(); + value.reserve(array.Size()); + for (auto& image : array) + { + KernelCacheImage img = KernelCacheImage::LoadFromValue(image); + value.push_back(img); + } +} + + +void Serialize(SerializationContext& context, const std::vector<std::pair<uint64_t, uint64_t>>& value) +{ + context.writer.StartArray(); + for (const auto& pair : value) + { + context.writer.StartArray(); + Serialize(context, pair.first); + Serialize(context, pair.second); + context.writer.EndArray(); + } + context.writer.EndArray(); +} + +void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, uint64_t>>& value) +{ + auto array = context.doc[name.data()].GetArray(); + value.reserve(array.Size()); + for (auto& pair : array) + { + auto pair_array = pair.GetArray(); + value.push_back({pair_array[0].GetUint64(), pair_array[1].GetUint64()}); + } +} + +void Serialize(SerializationContext& context, const std::unordered_map<uint64_t, KernelCacheMachOHeader>& value) +{ + context.writer.StartArray(); + for (const auto& pair : value) + { + context.writer.StartArray(); + Serialize(context, pair.first); + Serialize(context, pair.second); + context.writer.EndArray(); + } + context.writer.EndArray(); +} + +void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<uint64_t, KernelCacheMachOHeader>& value) +{ + auto array = context.doc[name.data()].GetArray(); + for (auto& pair : array) + { + auto pair_array = pair.GetArray(); + uint64_t key = pair_array[0].GetUint64(); + KernelCacheMachOHeader header = KernelCacheMachOHeader::LoadFromValue(pair_array[1]); + value[key] = header; + } +} + + +void Serialize(SerializationContext& context, const std::unordered_map<uint64_t, KernelCacheImage> value) +{ + context.writer.StartArray(); + for (const auto& pair : value) + { + context.writer.StartArray(); + Serialize(context, pair.first); + Serialize(context, pair.second); + context.writer.EndArray(); + } + context.writer.EndArray(); +} + +void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<uint64_t, KernelCacheImage>& value) +{ + auto array = context.doc[name.data()].GetArray(); + for (auto& pair : array) + { + auto pair_array = pair.GetArray(); + uint64_t key = pair_array[0].GetUint64(); + KernelCacheImage image = KernelCacheImage::LoadFromValue(pair_array[1]); + value[key] = image; + } +} + + +void KernelCache::CacheInfo::Store(SerializationContext& context) const +{ + Serialize(context, "metadataVersion", METADATA_VERSION); + + MSS(images); + MSS(imageStarts); + MSS_CAST(cacheFormat, uint8_t); +} + +// static +std::optional<KernelCache::CacheInfo> KernelCache::CacheInfo::Load(DeserializationContext& context) +{ + if (!context.doc.HasMember("metadataVersion")) + { + LogError("Shared Cache metadata version missing"); + return std::nullopt; + } + + if (context.doc["metadataVersion"].GetUint() != METADATA_VERSION) + { + LogError("Shared Cache metadata version mismatch"); + return std::nullopt; + } + + CacheInfo cacheInfo; + cacheInfo.MSL(images); + cacheInfo.MSL(imageStarts); + cacheInfo.MSL_CAST(cacheFormat, uint8_t, KernelCacheFormat); + return cacheInfo; +} +void State::Store(SerializationContext& context, std::optional<KCViewState> viewState) const +{ + MSS(loadedImages); + MSS(viewState); +} + +void KernelCache::ModifiedState::Store(SerializationContext& context) const +{ + State::Store(context, viewState); +} + +KernelCache::ModifiedState KernelCache::ModifiedState::Load(DeserializationContext& context) +{ + KernelCache::ModifiedState state; + state.MSL(loadedImages); + state.MSL(viewState); + return state; +} + +KernelCache::ModifiedState KernelCache::ModifiedState::LoadAll(BinaryNinja::BinaryView *dscView, const CacheInfo& cacheInfo) +{ + uint64_t stateCount = dscView->GetUIntMetadata(KernelCacheMetadata::ModifiedStateCountTag); + KernelCache::ModifiedState state; + for (uint64_t i = 0; i < stateCount; ++i) + { + std::string key = KernelCacheMetadata::ModifiedStateTagPrefix + std::to_string(i); + std::string serialized = dscView->GetStringMetadata(key); + auto thisState = KernelCache::ModifiedState::LoadFromString(serialized); + state.Merge(std::move(thisState)); + } + return state; +} + +void KernelCache::ModifiedState::Merge(KernelCache::ModifiedState&& newer) +{ + loadedImages.merge(newer.loadedImages); + + if (newer.viewState) + viewState = newer.viewState; +} + +void KernelCacheImage::Store(SerializationContext& context) const +{ + MSS(installName); + MSS(headerFileLocation); + MSS(regions); +} + +// static +KernelCacheImage KernelCacheImage::Load(DeserializationContext& context) +{ + KernelCacheImage cacheImage; + cacheImage.MSL(installName); + cacheImage.MSL(headerFileLocation); + cacheImage.MSL(regions); + return cacheImage; +} + + +} // namespace KernelCacheCore diff --git a/view/kernelcache/core/KernelCache.h b/view/kernelcache/core/KernelCache.h new file mode 100644 index 00000000..8d10a4ec --- /dev/null +++ b/view/kernelcache/core/KernelCache.h @@ -0,0 +1,360 @@ +// +// Created by kat on 5/19/23. +// + +#include <binaryninjaapi.h> +#include "KCView.h" +#include "view/macho/machoview.h" +#include "MetadataSerializable.hpp" +#include "../api/kernelcachecore.h" + +#ifndef KERNELCACHE_KERNELCACHE_H +#define KERNELCACHE_KERNELCACHE_H + +DECLARE_KERNELCACHE_API_OBJECT(BNKernelCache, KernelCache); + +namespace KernelCacheCore { + + enum KCViewState + { + KCViewStateUnloaded, + KCViewStateLoaded, + KCViewStateLoadedWithImages, + }; + + const std::string KernelCacheMetadataTag = "KERNELCACHE-KernelCacheData"; + + struct MemoryRegion : public MetadataSerializable<MemoryRegion> + { + enum class Type + { + Image, + NonImage, + }; + + std::string prettyName; + uint64_t start; + uint64_t size; + uint64_t fileOffset; + BNSegmentFlag flags; + Type type; + + void Store(SerializationContext& context) const + { + MSS(prettyName); + MSS(start); + MSS(size); + MSS(fileOffset); + MSS_CAST(flags, uint64_t); + MSS_CAST(type, uint8_t); + } + + static MemoryRegion Load(DeserializationContext& context) + { + MemoryRegion region; + region.MSL(prettyName); + region.MSL(start); + region.MSL(size); + region.MSL(fileOffset); + region.MSL_CAST(flags, uint64_t, BNSegmentFlag); + region.MSL_CAST(type, uint8_t, Type); + return region; + } + }; + + struct KernelCacheImage : public MetadataSerializable<KernelCacheImage> { + std::string installName; + uint64_t headerFileLocation; + std::vector<MemoryRegion> regions; + + void Store(SerializationContext& context) const; + static KernelCacheImage Load(DeserializationContext& context); + }; + + #if defined(__GNUC__) || defined(__clang__) + #define PACKED_STRUCT __attribute__((packed)) + #else + #define PACKED_STRUCT + #endif + + #if defined(_MSC_VER) + #pragma pack(push, 1) + #else + + #endif + + #if defined(_MSC_VER) + #pragma pack(pop) + #else + + #endif + + using namespace BinaryNinja; + struct KernelCacheMachOHeader : public MetadataSerializable<KernelCacheMachOHeader> + { + uint64_t textBase = 0; + uint64_t textBaseFileOffset = 0; + uint64_t loadCommandOffset = 0; + mach_header_64 ident; + std::string identifierPrefix; + std::string installName; + + std::vector<std::pair<uint64_t, bool>> entryPoints; + std::vector<uint64_t> m_entryPoints; // list of entrypoints + + symtab_command symtab; + dysymtab_command dysymtab; + dyld_info_command dyldInfo; + routines_command_64 routines64; + function_starts_command functionStarts; + std::vector<section_64> moduleInitSections; + std::vector<section_64> moduleTermSections; + linkedit_data_command exportTrie; + linkedit_data_command chainedFixups {}; + + uint64_t relocationBase; + // Section and program headers, internally use 64-bit form as it is a superset of 32-bit + std::vector<segment_command_64> segments; // only three types of sections __TEXT, __DATA, __IMPORT + segment_command_64 linkeditSegment; + std::vector<section_64> sections; + std::vector<std::string> sectionNames; + + std::vector<section_64> symbolStubSections; + std::vector<section_64> symbolPointerSections; + + std::vector<std::string> dylibs; + + build_version_command buildVersion; + std::vector<build_tool_version> buildToolVersions; + + bool linkeditPresent = false; + bool dysymPresent = false; + bool dyldInfoPresent = false; + bool exportTriePresent = false; + bool chainedFixupsPresent = false; + bool routinesPresent = false; + bool functionStartsPresent = false; + bool relocatable = false; + + void Store(SerializationContext& context) const { + MSS(textBase); + MSS(textBaseFileOffset); + MSS(loadCommandOffset); + MSS_SUBCLASS(ident); + MSS(identifierPrefix); + MSS(installName); + MSS(entryPoints); + MSS(m_entryPoints); + MSS_SUBCLASS(symtab); + MSS_SUBCLASS(dysymtab); + MSS_SUBCLASS(dyldInfo); + MSS_SUBCLASS(routines64); + MSS_SUBCLASS(functionStarts); + MSS_SUBCLASS(moduleInitSections); + MSS_SUBCLASS(moduleTermSections); + MSS_SUBCLASS(exportTrie); + MSS_SUBCLASS(chainedFixups); + MSS(relocationBase); + MSS_SUBCLASS(segments); + MSS_SUBCLASS(linkeditSegment); + MSS_SUBCLASS(sections); + MSS(sectionNames); + MSS_SUBCLASS(symbolStubSections); + MSS_SUBCLASS(symbolPointerSections); + MSS(dylibs); + MSS_SUBCLASS(buildVersion); + MSS_SUBCLASS(buildToolVersions); + MSS(linkeditPresent); + MSS(dysymPresent); + MSS(dyldInfoPresent); + MSS(exportTriePresent); + MSS(chainedFixupsPresent); + MSS(routinesPresent); + MSS(functionStartsPresent); + MSS(relocatable); + } + + static KernelCacheMachOHeader Load(DeserializationContext& context) { + KernelCacheMachOHeader header; + header.MSL(textBase); + header.MSL(textBaseFileOffset); + header.MSL(loadCommandOffset); + header.MSL(ident); + header.MSL(identifierPrefix); + header.MSL(installName); + header.MSL(entryPoints); + header.MSL(m_entryPoints); + header.MSL(symtab); + header.MSL(dysymtab); + header.MSL(dyldInfo); + header.MSL(routines64); + header.MSL(functionStarts); + header.MSL(moduleInitSections); + header.MSL(moduleTermSections); + header.MSL(exportTrie); + header.MSL(chainedFixups); + header.MSL(relocationBase); + header.MSL(segments); + header.MSL(linkeditSegment); + header.MSL(sections); + header.MSL(sectionNames); + header.MSL(symbolStubSections); + header.MSL(symbolPointerSections); + header.MSL(dylibs); + header.MSL(buildVersion); + header.MSL(buildToolVersions); + header.MSL(linkeditPresent); + header.MSL(dysymPresent); + header.MSL(dyldInfoPresent); + header.MSL(exportTriePresent); + header.MSL(chainedFixupsPresent); + header.MSL(routinesPresent); + header.MSL(functionStartsPresent); + header.MSL(relocatable); + return header; + } + }; + + class KernelCache : public MetadataSerializable<KernelCache> + { + IMPLEMENT_KERNELCACHE_API_OBJECT(BNKernelCache); + + std::atomic<int> m_refs = 0; + + public: + virtual void AddRef() { m_refs.fetch_add(1); } + + virtual void Release() + { + // undo actions will lock a file lock we hold and then wait for main thread + // so we need to release the ref later. + WorkerPriorityEnqueue([this]() { + if (m_refs.fetch_sub(1) == 1) + delete this; + }); + } + + virtual void AddAPIRef() { AddRef(); } + + virtual void ReleaseAPIRef() { Release(); } + + public: + enum KernelCacheFormat + { + FilesetCacheFormat, + PrelinkedCacheFormat, + }; + + struct CacheInfo; + struct ModifiedState; + + struct ViewSpecificState; + + void Store(SerializationContext& context) const; + void Load(DeserializationContext& context); + + private: + Ref<Logger> m_logger; + /* VIEW STATE BEGIN -- SERIALIZE ALL OF THIS AND STORE IT IN RAW VIEW */ + + // State that is initialized during `PerformInitialLoad` and does + // not change thereafter. + std::shared_ptr<const CacheInfo> m_cacheInfo; + + // Protects member variables below. + mutable std::mutex m_mutex; + + // State that has been modified since this instance was created + // or last saved to the view-specific state. + // To get an accurate view of the current state, both these modifications + // and the view-specific state must be consulted. + std::unique_ptr<ModifiedState> m_modifiedState; + + // Serialized once by PerformInitialLoad and available after m_viewState == Loaded + bool m_metadataValid = false; + + /* API VIEW START */ + BinaryNinja::Ref<BinaryNinja::BinaryView> m_kcView; + /* API VIEW END */ + + std::shared_ptr<ViewSpecificState> m_viewSpecificState; + + private: + void PerformInitialLoad(std::lock_guard<std::mutex>&); + void DeserializeFromRawView(std::lock_guard<std::mutex>&); + + public: + static KernelCache* GetFromKCView(BinaryNinja::Ref<BinaryNinja::BinaryView> kcView); + static uint64_t FastGetImageCount(BinaryNinja::Ref<BinaryNinja::BinaryView> kcView); + bool SaveCacheInfoToKCView(std::lock_guard<std::mutex>&); + bool SaveModifiedStateToKCView(std::lock_guard<std::mutex>&); + std::optional<uint64_t> GetImageStart(std::string installName); + std::optional<KernelCacheMachOHeader> HeaderForVMAddress(uint64_t address); + std::optional<KernelCacheMachOHeader> HeaderForFileAddress(uint64_t address); + bool LoadImageWithInstallName(std::lock_guard<std::mutex>& lock, std::string installName); + bool LoadImageWithInstallName(std::string installName); + bool LoadImageContainingAddress(std::lock_guard<std::mutex>& lock, uint64_t address); + bool LoadImageContainingAddress(uint64_t address); + std::string NameForAddress(uint64_t address); + std::string ImageNameForAddress(uint64_t address); + std::vector<std::string> GetAvailableImages(); + std::vector<KernelCacheImage> GetLoadedImages(); + + std::vector<std::pair<uint64_t, std::pair<std::string, std::string>>> LoadAllSymbolsAndWait(); + + const std::unordered_map<std::string, uint64_t>& AllImageStarts() const; + const std::unordered_map<uint64_t, KernelCacheMachOHeader> AllImageHeaders() const; + + std::string SerializedImageHeaderForVMAddress(uint64_t address); + std::string SerializedImageHeaderForName(std::string name); + + KCViewState ViewState() const; + + explicit KernelCache(BinaryNinja::Ref<BinaryNinja::BinaryView> rawView); + virtual ~KernelCache(); + + static bool InitializeSegmentsForHeader(Ref<BinaryView> view, const KernelCacheMachOHeader& header, const KernelCacheImage& targetImage); + static std::optional<KernelCacheMachOHeader> LoadHeaderForAddress(Ref<BinaryView> view, uint64_t address, std::string installName); + static void InitializeHeader(Ref<BinaryView> view, KernelCacheMachOHeader header); + static void ReadExportNode(Ref<BinaryView> view, std::vector<Ref<Symbol>>& symbolList, KernelCacheMachOHeader& header, DataBuffer& buffer, + uint64_t textBase, const std::string& currentText, size_t cursor, uint32_t endGuard); + static std::vector<Ref<Symbol>> ParseExportTrie(Ref<BinaryView> view, KernelCacheMachOHeader header); + static std::vector<std::pair<uint64_t, std::pair<BNSymbolType, std::string>>> ParseSymbolTable(Ref<BinaryView> view, KernelCacheMachOHeader header, bool defineSymbolsInView = true); + }; + + + class KernelCacheMetadata + { + public: + static std::optional<KernelCacheMetadata> LoadFromView(BinaryView*); + static bool ViewHasMetadata(BinaryView*); + + std::string InstallNameForImageBaseAddress(uint64_t baseAddress) const; + + std::vector<KernelCacheImage> LoadedImages(); + + ~KernelCacheMetadata(); + KernelCacheMetadata(KernelCacheMetadata&&); + KernelCacheMetadata& operator=(KernelCacheMetadata&&); + + private: + KernelCacheMetadata(KernelCache::CacheInfo, KernelCache::ModifiedState); + + std::unique_ptr<KernelCache::CacheInfo> cacheInfo; + std::unique_ptr<KernelCache::ModifiedState> state; + + friend struct KernelCache::ModifiedState; + friend class KernelCache; + + static const std::string Tag; + static const std::string CacheInfoTag; + static const std::string ModifiedStateTagPrefix; + static const std::string ModifiedStateCountTag; + }; + +} + +void InitKernelcache(); + +#endif //KERNELCACHE_KERNELCACHE_H + diff --git a/view/kernelcache/core/MetadataSerializable.cpp b/view/kernelcache/core/MetadataSerializable.cpp new file mode 100644 index 00000000..dea6243e --- /dev/null +++ b/view/kernelcache/core/MetadataSerializable.cpp @@ -0,0 +1,552 @@ +#include "MetadataSerializable.hpp" + +namespace KernelCacheCore { + + void Serialize(SerializationContext& context, std::string_view str) { + context.writer.String(str.data(), str.length()); + } + + void Serialize(SerializationContext& context, const char* value) { + Serialize(context, std::string_view(value)); + } + + void Serialize(SerializationContext& context, bool b) + { + context.writer.Bool(b); + } + + void Serialize(SerializationContext& context, int8_t value) { + context.writer.Int(value); + } + + void Serialize(SerializationContext& context, uint8_t value) { + context.writer.Uint(value); + } + + void Serialize(SerializationContext& context, int16_t value) { + context.writer.Int(value); + } + + void Serialize(SerializationContext& context, uint16_t value) { + context.writer.Uint(value); + } + + void Serialize(SerializationContext& context, int32_t value) { + context.writer.Int(value); + } + + void Serialize(SerializationContext& context, uint32_t value) { + context.writer.Uint(value); + } + + void Serialize(SerializationContext& context, int64_t value) { + context.writer.Int64(value); + } + + void Serialize(SerializationContext& context, uint64_t value) { + context.writer.Uint64(value); + } + + void Deserialize(DeserializationContext& context, std::string_view name, bool& b) { + b = context.doc[name.data()].GetBool(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, uint8_t& b) + { + b = static_cast<uint8_t>(context.doc[name.data()].GetUint64()); + } + + void Deserialize(DeserializationContext& context, std::string_view name, uint16_t& b) + { + b = static_cast<uint16_t>(context.doc[name.data()].GetUint64()); + } + + void Deserialize(DeserializationContext& context, std::string_view name, uint32_t& b) + { + b = static_cast<uint32_t>(context.doc[name.data()].GetUint64()); + } + + void Deserialize(DeserializationContext& context, std::string_view name, uint64_t& b) + { + b = context.doc[name.data()].GetUint64(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, int8_t& b) + { + b = context.doc[name.data()].GetInt64(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, int16_t& b) + { + b = context.doc[name.data()].GetInt64(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, int32_t& b) + { + b = context.doc[name.data()].GetInt(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, int64_t& b) + { + b = context.doc[name.data()].GetInt64(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::string& b) + { + b = context.doc[name.data()].GetString(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::map<uint64_t, std::string>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + b[i.GetArray()[0].GetUint64()] = i.GetArray()[1].GetString(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<uint64_t, std::string>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + b[i.GetArray()[0].GetUint64()] = i.GetArray()[1].GetString(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<uint64_t, uint64_t>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + b[i.GetArray()[0].GetUint64()] = i.GetArray()[1].GetUint64(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<std::string, std::unordered_map<uint64_t, uint64_t>>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + { + std::string key = i.GetArray()[0].GetString(); + std::unordered_map<uint64_t, uint64_t> memArray; + for (auto& member : i.GetArray()[1].GetArray()) + { + memArray[member.GetArray()[0].GetUint64()] = member.GetArray()[1].GetUint64(); + } + b[key] = memArray; + } + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<std::string, std::string>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + b[i.GetArray()[0].GetString()] = i.GetArray()[1].GetString(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::string>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + b.emplace_back(i.GetString()); + } + + // Note: This flattens the pair into [first, second.first, second.second] with no nested arrays. + void Serialize(SerializationContext& context, const std::pair<uint64_t, std::pair<uint64_t, uint64_t>>& value) + { + context.writer.StartArray(); + Serialize(context, value.first); + Serialize(context, value.second.first); + Serialize(context, value.second.second); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, std::pair<uint64_t, uint64_t>>>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + { + std::pair<uint64_t, std::pair<uint64_t, uint64_t>> j; + j.first = i.GetArray()[0].GetUint64(); + j.second.first = i.GetArray()[1].GetUint64(); + j.second.second = i.GetArray()[2].GetUint64(); + b.push_back(j); + } + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, bool>>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + { + std::pair<uint64_t, bool> j; + j.first = i.GetArray()[0].GetUint64(); + j.second = i.GetArray()[1].GetBool(); + b.push_back(j); + } + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<uint64_t>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + { + b.push_back(i.GetUint64()); + } + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<std::string, uint64_t>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + { + b[i.GetArray()[0].GetString()] = i.GetArray()[1].GetUint64(); + } + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, std::vector<std::pair<uint64_t, std::string>>>>& b) + { + for (auto& i : context.doc[name.data()].GetArray()) + { + std::pair<uint64_t, std::vector<std::pair<uint64_t, std::string>>> j; + j.first = i.GetArray()[0].GetUint64(); + for (auto& k : i.GetArray()[1].GetArray()) + { + j.second.push_back({k.GetArray()[0].GetUint64(), k.GetArray()[1].GetString()}); + } + b.push_back(j); + } + } + + void Serialize(SerializationContext& context, const mach_header_64& value) { + context.writer.StartArray(); + Serialize(context, value.magic); + // cputype and cpusubtype are signed but were serialized as unsigned in + // v4.2 (metadata version 2). We continue serializing them as unsigned + // so we don't need to bump the metadata version. + Serialize(context, static_cast<uint32_t>(value.cputype)); + Serialize(context, static_cast<uint32_t>(value.cpusubtype)); + Serialize(context, value.filetype); + Serialize(context, value.ncmds); + Serialize(context, value.sizeofcmds); + Serialize(context, value.flags); + Serialize(context, value.reserved); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, mach_header_64& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.magic = bArr[0].GetUint(); + b.cputype = bArr[1].GetUint(); + b.cpusubtype = bArr[2].GetUint(); + b.filetype = bArr[3].GetUint(); + b.ncmds = bArr[4].GetUint(); + b.sizeofcmds = bArr[5].GetUint(); + b.flags = bArr[6].GetUint(); + b.reserved = bArr[7].GetUint(); + } + + void Serialize(SerializationContext& context, const symtab_command& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.symoff); + Serialize(context, value.nsyms); + Serialize(context, value.stroff); + Serialize(context, value.strsize); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, symtab_command& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.symoff = bArr[2].GetUint(); + b.nsyms = bArr[3].GetUint(); + b.stroff = bArr[4].GetUint(); + b.strsize = bArr[5].GetUint(); + } + + void Serialize(SerializationContext& context, const dysymtab_command& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.ilocalsym); + Serialize(context, value.nlocalsym); + Serialize(context, value.iextdefsym); + Serialize(context, value.nextdefsym); + Serialize(context, value.iundefsym); + Serialize(context, value.nundefsym); + Serialize(context, value.tocoff); + Serialize(context, value.ntoc); + Serialize(context, value.modtaboff); + Serialize(context, value.nmodtab); + Serialize(context, value.extrefsymoff); + Serialize(context, value.nextrefsyms); + Serialize(context, value.indirectsymoff); + Serialize(context, value.nindirectsyms); + Serialize(context, value.extreloff); + Serialize(context, value.nextrel); + Serialize(context, value.locreloff); + Serialize(context, value.nlocrel); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, dysymtab_command& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.ilocalsym = bArr[2].GetUint(); + b.nlocalsym = bArr[3].GetUint(); + b.iextdefsym = bArr[4].GetUint(); + b.nextdefsym = bArr[5].GetUint(); + b.iundefsym = bArr[6].GetUint(); + b.nundefsym = bArr[7].GetUint(); + b.tocoff = bArr[8].GetUint(); + b.ntoc = bArr[9].GetUint(); + b.modtaboff = bArr[10].GetUint(); + b.nmodtab = bArr[11].GetUint(); + b.extrefsymoff = bArr[12].GetUint(); + b.nextrefsyms = bArr[13].GetUint(); + b.indirectsymoff = bArr[14].GetUint(); + b.nindirectsyms = bArr[15].GetUint(); + b.extreloff = bArr[16].GetUint(); + b.nextrel = bArr[17].GetUint(); + b.locreloff = bArr[18].GetUint(); + b.nlocrel = bArr[19].GetUint(); + } + + void Serialize(SerializationContext& context, const dyld_info_command& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.rebase_off); + Serialize(context, value.rebase_size); + Serialize(context, value.bind_off); + Serialize(context, value.bind_size); + Serialize(context, value.weak_bind_off); + Serialize(context, value.weak_bind_size); + Serialize(context, value.lazy_bind_off); + Serialize(context, value.lazy_bind_size); + Serialize(context, value.export_off); + Serialize(context, value.export_size); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, dyld_info_command& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.rebase_off = bArr[2].GetUint(); + b.rebase_size = bArr[3].GetUint(); + b.bind_off = bArr[4].GetUint(); + b.bind_size = bArr[5].GetUint(); + b.weak_bind_off = bArr[6].GetUint(); + b.weak_bind_size = bArr[7].GetUint(); + b.lazy_bind_off = bArr[8].GetUint(); + b.lazy_bind_size = bArr[9].GetUint(); + b.export_off = bArr[10].GetUint(); + b.export_size = bArr[11].GetUint(); + } + + void Serialize(SerializationContext& context, const routines_command_64& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.init_address); + Serialize(context, value.init_module); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, routines_command_64& b) + { + auto bArr = context.doc[name.data()].GetArray(); + // Because we might open databases that had not previously serialized this, we must allow + // an empty array, otherwise we will crash! + if (bArr.Size() < 4) + return; + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.init_address = bArr[2].GetUint64(); + b.init_module = bArr[3].GetUint64(); + } + + void Serialize(SerializationContext& context, const function_starts_command& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.funcoff); + Serialize(context, value.funcsize); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, function_starts_command& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.funcoff = bArr[2].GetUint(); + b.funcsize = bArr[3].GetUint(); + } + + void Serialize(SerializationContext& context, const section_64& value) + { + context.writer.StartArray(); + + std::string_view sectname(value.sectname, 16); + std::string_view segname(value.segname, 16); + + Serialize(context, sectname.substr(0, sectname.find('\0'))); + Serialize(context, segname.substr(0, segname.find('\0'))); + Serialize(context, value.addr); + Serialize(context, value.size); + Serialize(context, value.offset); + Serialize(context, value.align); + Serialize(context, value.reloff); + Serialize(context, value.nreloc); + Serialize(context, value.flags); + Serialize(context, value.reserved1); + Serialize(context, value.reserved2); + Serialize(context, value.reserved3); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<section_64>& b) + { + auto bArr = context.doc[name.data()].GetArray(); + for (auto& s : bArr) + { + section_64 sec; + auto s2 = s.GetArray(); + std::string sectNameStr = s2[0].GetString(); + memset(sec.sectname, 0, 16); + memcpy(sec.sectname, sectNameStr.c_str(), sectNameStr.size()); + std::string segNameStr = s2[1].GetString(); + memset(sec.segname, 0, 16); + memcpy(sec.segname, segNameStr.c_str(), segNameStr.size()); + sec.addr = s2[2].GetUint64(); + sec.size = s2[3].GetUint64(); + sec.offset = s2[4].GetUint(); + sec.align = s2[5].GetUint(); + sec.reloff = s2[6].GetUint(); + sec.nreloc = s2[7].GetUint(); + sec.flags = s2[8].GetUint(); + sec.reserved1 = s2[9].GetUint(); + sec.reserved2 = s2[10].GetUint(); + sec.reserved3 = s2[11].GetUint(); + b.push_back(std::move(sec)); + } + } + + void Serialize(SerializationContext& context, const linkedit_data_command& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.dataoff); + Serialize(context, value.datasize); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, linkedit_data_command& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.dataoff = bArr[2].GetUint(); + b.datasize = bArr[3].GetUint(); + } + + void Serialize(SerializationContext& context, const segment_command_64& value) + { + context.writer.StartArray(); + std::string_view segname(value.segname, 16); + Serialize(context, segname.substr(0, segname.find('\0'))); + Serialize(context, value.vmaddr); + Serialize(context, value.vmsize); + Serialize(context, value.fileoff); + Serialize(context, value.filesize); + Serialize(context, value.maxprot); + Serialize(context, value.initprot); + Serialize(context, value.nsects); + Serialize(context, value.flags); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, segment_command_64& b) + { + auto bArr = context.doc[name.data()].GetArray(); + std::string segNameStr = bArr[0].GetString(); + memset(b.segname, 0, 16); + memcpy(b.segname, segNameStr.c_str(), segNameStr.size()); + b.vmaddr = bArr[1].GetUint64(); + b.vmsize = bArr[2].GetUint64(); + b.fileoff = bArr[3].GetUint64(); + b.filesize = bArr[4].GetUint64(); + b.maxprot = bArr[5].GetUint(); + b.initprot = bArr[6].GetUint(); + b.nsects = bArr[7].GetUint(); + b.flags = bArr[8].GetUint(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<segment_command_64>& b) + { + auto bArr = context.doc[name.data()].GetArray(); + for (auto& s : bArr) + { + segment_command_64 sec; + auto s2 = s.GetArray(); + std::string segNameStr = s2[0].GetString(); + memset(sec.segname, 0, 16); + memcpy(sec.segname, segNameStr.c_str(), segNameStr.size()); + sec.vmaddr = s2[1].GetUint64(); + sec.vmsize = s2[2].GetUint64(); + sec.fileoff = s2[3].GetUint64(); + sec.filesize = s2[4].GetUint64(); + sec.maxprot = s2[5].GetUint(); + sec.initprot = s2[6].GetUint(); + sec.nsects = s2[7].GetUint(); + sec.flags = s2[8].GetUint(); + b.push_back(std::move(sec)); + } + } + + void Serialize(SerializationContext& context, const build_version_command& value) + { + context.writer.StartArray(); + Serialize(context, value.cmd); + Serialize(context, value.cmdsize); + Serialize(context, value.platform); + Serialize(context, value.minos); + Serialize(context, value.sdk); + Serialize(context, value.ntools); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, build_version_command& b) + { + auto bArr = context.doc[name.data()].GetArray(); + b.cmd = bArr[0].GetUint(); + b.cmdsize = bArr[1].GetUint(); + b.platform = bArr[2].GetUint(); + b.minos = bArr[3].GetUint(); + b.sdk = bArr[4].GetUint(); + b.ntools = bArr[5].GetUint(); + } + + void Serialize(SerializationContext& context, const build_tool_version& value) + { + context.writer.StartArray(); + Serialize(context, value.tool); + Serialize(context, value.version); + context.writer.EndArray(); + } + + void Deserialize(DeserializationContext& context, std::string_view name, std::vector<build_tool_version>& b) + { + auto bArr = context.doc[name.data()].GetArray(); + for (auto& s : bArr) + { + build_tool_version sec; + auto s2 = s.GetArray(); + sec.tool = s2[0].GetUint(); + sec.version = s2[1].GetUint(); + b.push_back(sec); + } + } + +} // namespace KernelCacheCore diff --git a/view/kernelcache/core/MetadataSerializable.hpp b/view/kernelcache/core/MetadataSerializable.hpp new file mode 100644 index 00000000..5397a56c --- /dev/null +++ b/view/kernelcache/core/MetadataSerializable.hpp @@ -0,0 +1,254 @@ +// +// Created by kat on 5/31/23. +// + +/* + * Welcome to, this file. + * + * This is a metadata serialization helper. + * + * Have you ever wished turning a complex datastructure into a Metadata object was as easy in C++ as it is in python? + * Do you like macros and templates? + * + * Great news. + * + * Implement these on your `public MetadataSerializable<T>` subclass: + * ``` + class MyClass : public MetadataSerializable<MyClass> { + void Store(SerializationContext& context) const { + MSS(m_someVariable); + MSS(m_someOtherVariable); + } + void Load(DeserializationContext& context) { + MSL(m_someVariable); + MSL(m_someOtherVariable); + } + } + ``` + * Then, you can turn your object into a Metadata object with `AsMetadata()`, and load it back with + `LoadFromMetadata()`. + * + * Serialized fields will be automatically repopulated. + * + * Other ser/deser formats (rapidjson objects, strings) also exist. You can use these to achieve nesting, but probably + avoid that. + * */ + +#ifndef KERNELCACHE_CORE_METADATASERIALIZABLE_HPP +#define KERNELCACHE_CORE_METADATASERIALIZABLE_HPP + +#include <cassert> +#include "binaryninjaapi.h" +#include "rapidjson/document.h" +#include "rapidjson/stringbuffer.h" +#include "rapidjson/prettywriter.h" +#include "../api/kernelcachecore.h" +#include "view/macho/machoview.h" + +using namespace BinaryNinja; + +namespace KernelCacheCore { + +#define MSS(name) context.store(#name, name) +#define MSS_CAST(name, type) context.store(#name, (type) name) +#define MSS_SUBCLASS(name) Serialize(context, #name, name) +#define MSL(name) name = context.load<decltype(name)>(#name) +#define MSL_CAST(name, storedType, type) name = (type)context.load<storedType>(#name) + + struct DeserializationContext; + + struct SerializationContext { + rapidjson::StringBuffer buffer; + rapidjson::Writer<rapidjson::StringBuffer> writer; + + SerializationContext() : buffer(), writer(buffer) { + } + + template <typename T> + void store(std::string_view x, const T& y) + { + Serialize(*this, x, y); + } + }; + + struct DeserializationContext { + rapidjson::Document doc; + + template <typename T> + T load(std::string_view x) + { + T value; + Deserialize(*this, x, value); + return value; + } + }; + + template <typename Derived, typename LoadResult = Derived> + class MetadataSerializable { + public: + template <typename... Args> + std::string AsString(Args&&... args) const { + SerializationContext context; + Store(context, std::forward<Args>(args)...); + + return context.buffer.GetString(); + } + + static LoadResult LoadFromString(const std::string& s) { + DeserializationContext context; + [[maybe_unused]] rapidjson::ParseResult result = context.doc.Parse(s.c_str()); + assert(result); + return Derived::Load(context); + } + + static LoadResult LoadFromValue(rapidjson::Value& s) { + DeserializationContext context; + context.doc.CopyFrom(s, context.doc.GetAllocator()); + return Derived::Load(context); + } + + template <typename... Args> + Ref<Metadata> AsMetadata(Args&&... args) const { + return new Metadata(AsString(std::forward<Args>(args)...)); + } + + template <typename... Args> + void Store(SerializationContext& context, Args&&... args) const { + context.writer.StartObject(); + AsDerived().Store(context, std::forward<Args>(args)...); + context.writer.EndObject(); + } + + private: + const Derived& AsDerived() const { return static_cast<const Derived&>(*this); } + Derived& AsDerived() { return static_cast<Derived&>(*this); } + }; + + // The functions below are not part of the FFI API, but are exported so they can be shared with sharedcacheui. + + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, std::string_view str); + + template <typename T> + inline void Serialize(SerializationContext& context, const MetadataSerializable<T>& value) + { + value.Store(context); + } + + template <typename T> + inline void Serialize(SerializationContext& context, std::string_view name, const T& value) + { + Serialize(context, name); + Serialize(context, value); + } + + template <typename First, typename Second> + void Serialize(SerializationContext& context, const std::pair<First, Second>& value) + { + context.writer.StartArray(); + Serialize(context, value.first); + Serialize(context, value.second); + context.writer.EndArray(); + } + + template <typename K, typename V, typename L> + void Serialize(SerializationContext& context, const std::map<K, V, L>& value) + { + context.writer.StartArray(); + for (auto& pair : value) + { + Serialize(context, pair); + } + context.writer.EndArray(); + } + + template <typename K, typename V> + void Serialize(SerializationContext& context, const std::unordered_map<K, V>& value) + { + context.writer.StartArray(); + for (auto& pair : value) + { + Serialize(context, pair); + } + context.writer.EndArray(); + } + + template <typename T> + void Serialize(SerializationContext& context, const std::vector<T>& values) + { + context.writer.StartArray(); + for (const auto& value : values) + { + Serialize(context, value); + } + context.writer.EndArray(); + } + + template <typename T> + void Serialize(SerializationContext& context, const std::optional<T>& value) + { + if (value.has_value()) + Serialize(context, *value); + else + context.writer.Null(); + } + + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, const char*); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, bool b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, bool& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, uint8_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, uint8_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, uint16_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, uint16_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, uint32_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, uint32_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, uint64_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, uint64_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, int8_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, int8_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, int16_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, int16_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, int32_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, int32_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, int64_t b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, int64_t& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, std::string_view b); + KERNELCACHE_FFI_API void Serialize(SerializationContext& context, const std::pair<uint64_t, std::pair<uint64_t, uint64_t>>& value); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::string& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::map<uint64_t, std::string>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<uint64_t, std::string>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<uint64_t, uint64_t>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<std::string, std::unordered_map<uint64_t, uint64_t>>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<std::string, std::string>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::string>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, std::pair<uint64_t, uint64_t>>>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, bool>>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::vector<uint64_t>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::unordered_map<std::string, uint64_t>& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext& context, std::string_view name, std::vector<std::pair<uint64_t, std::vector<std::pair<uint64_t, std::string>>>>& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const mach_header_64& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, mach_header_64& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const symtab_command& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, symtab_command& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const dysymtab_command& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, dysymtab_command& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const dyld_info_command& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, dyld_info_command& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const routines_command_64& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, routines_command_64& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const function_starts_command& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, function_starts_command& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const section_64& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, std::vector<section_64>& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const linkedit_data_command& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, linkedit_data_command& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const segment_command_64& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, segment_command_64& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, std::vector<segment_command_64>& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const build_version_command& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, build_version_command& b); + KERNELCACHE_FFI_API void Serialize(SerializationContext&, const build_tool_version& b); + KERNELCACHE_FFI_API void Deserialize(DeserializationContext&, std::string_view name, std::vector<build_tool_version>& b); + +} // namespace SharedCacheCore + +#endif // KERNELCACHE_METADATASERIALIZABLE_HPP diff --git a/view/kernelcache/core/transformers/KernelCacheTransforms.cpp b/view/kernelcache/core/transformers/KernelCacheTransforms.cpp new file mode 100644 index 00000000..ac9ca375 --- /dev/null +++ b/view/kernelcache/core/transformers/KernelCacheTransforms.cpp @@ -0,0 +1,67 @@ +// +// kat // 11/8/22. +// + +#include <binaryninjaapi.h> + +using namespace BinaryNinja; + +#include "KernelCacheTransforms.h" +#include "libDER/libDER.h" +#include "libimg4/img4.h" +#include "liblzfse/lzfse.h" + +class IMG4PayloadTransform : public Transform +{ + +public: + IMG4PayloadTransform(): + Transform(DecodeTransform, "IMG4-Unencrypted", "IMG4-Unencrypted", "IMG4") + { + } + + virtual bool Decode(const DataBuffer& input, DataBuffer& output, const std::map<std::string, DataBuffer>& params) + { + DERItem* item = new DERItem; + item->data = (DERByte *)input.GetData(); + item->length = input.GetLength(); + + Img4Payload *payload = new Img4Payload; + DERImg4DecodePayload(item, payload); + + output = DataBuffer(payload->payload.data, payload->payload.length); + + return true; + } +}; + +class LZFSETransform : public Transform +{ + +public: + LZFSETransform(): + Transform(DecodeTransform, "LZFSE", "LZFSE", "Compression") + { + } + + virtual bool Decode(const DataBuffer& input, DataBuffer& output, const std::map<std::string, DataBuffer>& params) + { + size_t outputBufferSize = input.GetLength() * 6; + uint8_t* lzfseOutputBuffer = (uint8_t *)malloc(outputBufferSize); + uint8_t* scratchBuffer = (uint8_t *)malloc(lzfse_decode_scratch_size()); + size_t outSize = lzfse_decode_buffer(lzfseOutputBuffer, outputBufferSize, + (uint8_t *)input.GetData(), input.GetLength(), scratchBuffer); + + free(scratchBuffer); + + output = DataBuffer(lzfseOutputBuffer, outSize); + + return true; + } +}; + + +void RegisterTransformers() { + Transform::Register(new IMG4PayloadTransform()); + Transform::Register(new LZFSETransform()); +}
\ No newline at end of file diff --git a/view/kernelcache/core/transformers/KernelCacheTransforms.h b/view/kernelcache/core/transformers/KernelCacheTransforms.h new file mode 100644 index 00000000..65c97056 --- /dev/null +++ b/view/kernelcache/core/transformers/KernelCacheTransforms.h @@ -0,0 +1,18 @@ +// +// kat // 11/8/22. +// + +#ifndef KERNELCACHE_KERNELCACHETRANSFORMS_H +#define KERNELCACHE_KERNELCACHETRANSFORMS_H + +#ifdef __cplusplus +extern "C" { +#endif + +void RegisterTransformers(); + +#ifdef __cplusplus +} +#endif + +#endif //KERNELCACHE_KERNELCACHETRANSFORMS_H diff --git a/view/kernelcache/core/transformers/README.md b/view/kernelcache/core/transformers/README.md new file mode 100644 index 00000000..5cec5c89 --- /dev/null +++ b/view/kernelcache/core/transformers/README.md @@ -0,0 +1,3 @@ +## Transformers + +This directory contains decompressors and unwrappers that should allow dropping in an unextracted kernelcache directly into the product.
\ No newline at end of file diff --git a/view/kernelcache/core/transformers/libDER/DER_Decode.c b/view/kernelcache/core/transformers/libDER/DER_Decode.c new file mode 100644 index 00000000..3d08b616 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/DER_Decode.c @@ -0,0 +1,627 @@ +/* + * Copyright (c) 2005-2012 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +/* + * DER_Decode.c - DER decoding routines + */ +/* + * NOTICE: This file was modified by xerub to reflect binary code. + */ + +#include <libDER/DER_Decode.h> +#include <libDER/asn1Types.h> + +#include <libDER/libDER_config.h> + +#ifndef DER_DECODE_ENABLE +#error Please define DER_DECODE_ENABLE. +#endif + +#if DER_DECODE_ENABLE + +#define DER_DECODE_DEBUG 0 +#if DER_DECODE_DEBUG +#include <stdio.h> +#define derDecDbg(a) iprintf(a) +#define derDecDbg1(a, b) iprintf(a, b) +#define derDecDbg2(a, b, c) iprintf(a, b, c) +#define derDecDbg3(a, b, c, d) iprintf(a, b, c, d) +#else +#define derDecDbg(a) +#define derDecDbg1(a, b) +#define derDecDbg2(a, b, c) +#define derDecDbg3(a, b, c, d) +#endif /* DER_DECODE_DEBUG */ + +/* + * Basic decoding primitive. Only works with: + * + * -- definite length encoding + * -- one-byte tags + * -- max content length fits in a DERSize + * + * No malloc or copy of the contents is performed; the returned + * content->content.data is a pointer into the incoming der data. + */ +DERReturn DERDecodeItem( + const DERItem *der, /* data to decode */ + DERDecodedInfo *decoded) /* RETURNED */ +{ + return DERDecodeItemPartialBuffer(der, decoded, false); +} +DERReturn DERDecodeItemPartialBuffer( + const DERItem *der, /* data to decode */ + DERDecodedInfo *decoded, /* RETURNED */ + bool partial) +{ + DERByte tag1; /* first tag byte */ + DERByte len1; /* first length byte */ + DERTag tagNumber; /* tag number without class and method bits */ + DERByte *derPtr = der->data; + DERSize derLen = der->length; + + /* The tag decoding below is fully BER complient. We support a max tag + value of 2 ^ ((sizeof(DERTag) * 8) - 3) - 1 so for tag size 1 byte we + support tag values from 0 - 0x1F. For tag size 2 tag values + from 0 - 0x1FFF and for tag size 4 values from 0 - 0x1FFFFFFF. */ + if(derLen < 2) { + return DR_DecodeError; + } + /* Grab the first byte of the tag. */ + tag1 = *derPtr++; + derLen--; + tagNumber = tag1 & 0x1F; + if(tagNumber == 0x1F) { +#ifdef DER_MULTIBYTE_TAGS + if (*derPtr == 0x80 || *derPtr < 0x1F) { + return DR_DecodeError; + } + /* Long tag form: bit 8 of each octet shall be set to one unless it is + the last octet of the tag */ + const DERTag overflowMask = ((DERTag)0x7F << (sizeof(DERTag) * 8 - 7)); + DERByte tagByte; + tagNumber = 0; + do { + if(derLen < 2 || (tagNumber & overflowMask) != 0) { + return DR_DecodeError; + } + tagByte = *derPtr++; + derLen--; + tagNumber = (tagNumber << 7) | (tagByte & 0x7F); + } while((tagByte & 0x80) != 0); + + /* Check for any of the top 3 reserved bits being set. */ + if ((tagNumber & (overflowMask << 4)) != 0) +#endif + return DR_DecodeError; + } + /* Returned tag, top 3 bits are class/method remaining bits are number. */ + decoded->tag = ((DERTag)(tag1 & 0xE0) << ((sizeof(DERTag) - 1) * 8)) | tagNumber; + + /* Tag decoding above ensured we have at least one more input byte left. */ + len1 = *derPtr++; + derLen--; + if(len1 & 0x80) { + /* long length form - first byte is length of length */ + DERSize longLen = 0; /* long form length */ + unsigned dex; + + len1 &= 0x7f; + if((len1 > sizeof(DERSize)) || (len1 > derLen) || len1 == 0 || *derPtr == 0) { + /* no can do */ + return DR_DecodeError; + } + for(dex=0; dex<len1; dex++) { + longLen <<= 8; + longLen |= *derPtr++; + derLen--; + } + if(longLen > derLen && !partial) { + /* not enough data left for this encoding */ + return DR_DecodeError; + } + decoded->content.data = derPtr; + decoded->content.length = longLen; + } + else { + /* short length form, len1 is the length */ + if(len1 > derLen && !partial) { + /* not enough data left for this encoding */ + return DR_DecodeError; + } + decoded->content.data = derPtr; + decoded->content.length = len1; + } + + return DR_Success; + } + +/* + * Given a BIT_STRING, in the form of its raw content bytes, + * obtain the number of unused bits and the raw bit string bytes. + */ +DERReturn DERParseBitString( + const DERItem *contents, + DERItem *bitStringBytes, /* RETURNED */ + DERByte *numUnusedBits) /* RETURNED */ +{ + if(contents->length < 2) { + /* not enough room for actual bits after the unused bits field */ + *numUnusedBits = 0; + bitStringBytes->data = NULL; + bitStringBytes->length = 0; + return DR_Success; + } + *numUnusedBits = contents->data[0]; + bitStringBytes->data = contents->data + 1; + bitStringBytes->length = contents->length - 1; + return DR_Success; +} + +/* + * Given a BOOLEAN, in the form of its raw content bytes, + * obtain it's value. + */ +DERReturn DERParseBoolean( + const DERItem *contents, + bool *value) { /* RETURNED */ + if (contents->length != 1 || + (contents->data[0] != 0 && contents->data[0] != 0xFF)) + return DR_DecodeError; + + *value = contents->data[0] != 0; + return DR_Success; +} + +DERReturn DERParseInteger( + const DERItem *contents, + uint32_t *result) { /* RETURNED */ + uint64_t value; + DERReturn drtn = DERParseInteger64(contents, &value); + if (drtn) { + return drtn; + } + if (value >> 32) { + return DR_BufOverflow; + } + *result = value; + return DR_Success; +} + +DERReturn DERParseInteger64( + const DERItem *contents, + uint64_t *result) { /* RETURNED */ + const char *data = (char *)contents->data; + DERSize length = contents->length; + uint64_t value = 0; + + if (length == 0 || data[0] < 0) { + return DR_DecodeError; + } + + if (data[0] == 0) { + if (length >= 2) { + if (data[1] >= 0) { + return DR_DecodeError; + } + if (length > 9) { + return DR_BufOverflow; + } + } + } else if (length > 8) { + return DR_BufOverflow; + } + + while (length--) { + value <<= 8; + value += *(unsigned char *)data++; + } + *result = value; + return DR_Success; +} + +/* Sequence/set support */ + +/* + * To decode a set or sequence, call DERDecodeSeqInit once, then + * call DERDecodeSeqNext to get each enclosed item. + * DERDecodeSeqNext returns DR_EndOfSequence when no more + * items are available. + */ +DERReturn DERDecodeSeqInit( + const DERItem *der, /* data to decode */ + DERTag *tag, /* RETURNED tag of sequence/set. This will be + * either ASN1_CONSTR_SEQUENCE or ASN1_CONSTR_SET. */ + DERSequence *derSeq) /* RETURNED, to use in DERDecodeSeqNext */ +{ + DERDecodedInfo decoded; + DERReturn drtn; + + drtn = DERDecodeItem(der, &decoded); + if(drtn) { + return drtn; + } + *tag = decoded.tag; + switch(decoded.tag) { + case ASN1_CONSTR_SEQUENCE: + case ASN1_CONSTR_SET: + break; + default: + return DR_UnexpectedTag; + } + derSeq->nextItem = decoded.content.data; + derSeq->end = decoded.content.data + decoded.content.length; + return DR_Success; +} + +/* + * Use this to start in on decoding a sequence's content, when + * the top-level tag and content have already been decoded. + */ +DERReturn DERDecodeSeqContentInit( + const DERItem *content, + DERSequence *derSeq) /* RETURNED, to use in DERDecodeSeqNext */ +{ + /* just prepare for decoding items in content */ + derSeq->nextItem = content->data; + derSeq->end = content->data + content->length; + return DR_Success; +} + +DERReturn DERDecodeSeqNext( + DERSequence *derSeq, + DERDecodedInfo *decoded) /* RETURNED */ +{ + DERReturn drtn; + DERItem item; + + if(derSeq->nextItem >= derSeq->end) { + /* normal termination, contents all used up */ + return DR_EndOfSequence; + } + + /* decode next item */ + item.data = derSeq->nextItem; + item.length = derSeq->end - derSeq->nextItem; + drtn = DERDecodeItem(&item, decoded); + if(drtn) { + return drtn; + } + + /* skip over the item we just decoded */ + derSeq->nextItem = decoded->content.data + decoded->content.length; + return DR_Success; +} + +/* + * High level sequence parse, starting with top-level tag and content. + * Top level tag must be ASN1_CONSTR_SEQUENCE - if it's not, and that's + * OK, use DERParseSequenceContent(). + */ +DERReturn DERParseSequence( + const DERItem *der, + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + void *dest, /* DERDecodedInfo(s) here RETURNED */ + DERSize sizeToZero) /* optional */ +{ + DERReturn drtn; + DERDecodedInfo topDecode; + + drtn = DERDecodeItem(der, &topDecode); + if(drtn) { + return drtn; + } + if(topDecode.tag != ASN1_CONSTR_SEQUENCE) { + return DR_UnexpectedTag; + } + return DERParseSequenceContent(&topDecode.content, + numItems, itemSpecs, dest, sizeToZero); +} + +/* high level sequence parse, starting with sequence's content */ +DERReturn DERParseSequenceContent( + const DERItem *content, + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + void *dest, /* DERDecodedInfo(s) here RETURNED */ + DERSize sizeToZero) /* optional */ +{ + DERSequence derSeq; + DERReturn drtn; + DERShort itemDex; + DERByte *currDER; /* full DER encoding of current item */ + + if(sizeToZero) { + DERMemset(dest, 0, sizeToZero); + } + + drtn = DERDecodeSeqContentInit(content, &derSeq); + if(drtn) { + return drtn; + } + + /* main loop */ + for(itemDex=0 ; itemDex<numItems; ) { + DERDecodedInfo currDecoded; + DERShort i; + DERTag foundTag; + char foundMatch = 0; + + /* save this in case of DER_DEC_SAVE_DER */ + currDER = derSeq.nextItem; + + drtn = DERDecodeSeqNext(&derSeq, &currDecoded); + if(drtn) { + /* + * One legal error here is DR_EndOfSequence when + * all remaining DERSequenceItems are optional. + */ + if(drtn == DR_EndOfSequence) { + for(i=itemDex; i<numItems; i++) { + if(!(itemSpecs[i].options & DER_DEC_OPTIONAL)) { + /* unexpected end of sequence */ + return DR_IncompleteSeq; + } + } + /* the rest are optional; success */ + return DR_Success; + } + else { + /* any other error is fatal */ + return drtn; + } + } /* decode error */ + + /* + * Seek matching tag or ASN_ANY in itemSpecs, skipping + * over optional items. + */ + foundTag = currDecoded.tag; + derDecDbg1("--- foundTag 0x%x\n", foundTag); + + for(i=itemDex; i<numItems; i++) { + const DERItemSpec *currItemSpec = &itemSpecs[i]; + DERShort currOptions = currItemSpec->options; + derDecDbg3("--- currItem %u expectTag 0x%x currOptions 0x%x\n", + i, currItemSpec->tag, currOptions); + + if((currOptions & DER_DEC_ASN_ANY) || + (foundTag == currItemSpec->tag)) { + /* + * We're good with this one. Cook up destination address + * as appropriate. + */ + if(!(currOptions & DER_DEC_SKIP)) { + derDecDbg1("--- MATCH at currItem %u\n", i); + DERByte *byteDst = (DERByte *)dest + currItemSpec->offset; + DERItem *dst = (DERItem *)byteDst; + *dst = currDecoded.content; + if(currOptions & DER_DEC_SAVE_DER) { + /* recreate full DER encoding of this item */ + derDecDbg1("--- SAVE_DER at currItem %u\n", i); + dst->data = currDER; + dst->length += (currDecoded.content.data - currDER); + } + } + + /* on to next item */ + itemDex = i + 1; + + /* is this the end? */ + if(itemDex == numItems) { + /* normal termination if we consumed everything */ + if (derSeq.nextItem == derSeq.end) + return DR_Success; + else + return DR_DecodeError; + } + else { + /* on to next item */ + foundMatch = 1; + break; + } + } /* ASN_ANY, or match */ + + /* + * If current itemSpec isn't optional, abort - else on to + * next item + */ + if(!(currOptions & DER_DEC_OPTIONAL)) { + derDecDbg1("--- MISMATCH at currItem %u, !OPTIONAL, abort\n", i); + return DR_UnexpectedTag; + } + + /* else this was optional, on to next item */ + } /* searching for tag match */ + + if(foundMatch == 0) { + /* + * Found an item we couldn't match to any tag spec and we're at + * the end. + */ + derDecDbg("--- TAG NOT FOUND, abort\n"); + return DR_UnexpectedTag; + } + + /* else on to next item */ + } /* main loop */ + + /* + * If we get here, there appears to be more to process, but we've + * given the caller everything they want. + */ + return (derSeq.nextItem == derSeq.end) ? DR_Success : DR_DecodeError; +} + +#if 0 +/* + * High level sequence parse, starting with top-level tag and content. + * Top level tag must be ASN1_CONSTR_SEQUENCE - if it's not, and that's + * OK, use DERParseSequenceContent(). + */ +DERReturn DERParseSequenceOf( + const DERItem *der, + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + void *dest, /* DERDecodedInfo(s) here RETURNED */ + DERSize *numDestItems) /* output */ +{ + DERReturn drtn; + DERDecodedInfo topDecode; + + drtn = DERDecodeItem(der, &topDecode); + if(drtn) { + return drtn; + } + if(topDecode.tag != ASN1_CONSTR_SEQUENCE) { + return DR_UnexpectedTag; + } + return DERParseSequenceContent(&topDecode.content, + numItems, itemSpecs, dest, sizeToZero); +} + +/* + * High level set of parse, starting with top-level tag and content. + * Top level tag must be ASN1_CONSTR_SET - if it's not, and that's + * OK, use DERParseSetOrSequenceOfContent(). + */ +DERReturn DERParseSetOf( + const DERItem *der, + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + void *dest, /* DERDecodedInfo(s) here RETURNED */ + DERSize *numDestItems) /* output */ +{ + DERReturn drtn; + DERDecodedInfo topDecode; + + drtn = DERDecodeItem(der, &topDecode); + if(drtn) { + return drtn; + } + if(topDecode.tag != ASN1_CONSTR_SET) { + return DR_UnexpectedTag; + } + return DERParseSetOrSequenceOfContent(&topDecode.content, + numItems, itemSpecs, dest, numDestItems); +} + +/* High level set of or sequence of parse, starting with set or + sequence's content */ +DERReturn DERParseSetOrSequenceOfContent( + const DERItem *content, + void(*itemHandeler)(void *, const DERDecodedInfo *) + void *itemHandelerContext); +{ + DERSequence derSeq; + DERShort itemDex; + + drtn = DERDecodeSeqContentInit(content, &derSeq); + require_noerr_quiet(drtn, badCert); + + /* main loop */ + for (;;) { + DERDecodedInfo currDecoded; + DERShort i; + DERByte foundTag; + char foundMatch = 0; + + drtn = DERDecodeSeqNext(&derSeq, &currDecoded); + if(drtn) { + /* The only legal error here is DR_EndOfSequence. */ + if(drtn == DR_EndOfSequence) { + /* no more items left in the sequence; success */ + return DR_Success; + } + else { + /* any other error is fatal */ + require_noerr_quiet(drtn, badCert); + } + } /* decode error */ + + /* Each element can be anything. */ + foundTag = currDecoded.tag; + + /* + * We're good with this one. Cook up destination address + * as appropriate. + */ + DERByte *byteDst = (DERByte *)dest + currItemSpec->offset; + DERItem *dst = (DERItem *)byteDst; + *dst = currDecoded.content; + if(currOptions & DER_DEC_SAVE_DER) { + /* recreate full DER encoding of this item */ + derDecDbg1("--- SAVE_DER at currItem %u\n", i); + dst->data = currDER; + dst->length += (currDecoded.content.data - currDER); + } + + /* on to next item */ + itemDex = i + 1; + + /* is this the end? */ + if(itemDex == numItems) { + /* normal termination */ + return DR_Success; + } + else { + /* on to next item */ + foundMatch = 1; + break; + } + + /* + * If current itemSpec isn't optional, abort - else on to + * next item + */ + if(!(currOptions & DER_DEC_OPTIONAL)) { + derDecDbg1("--- MISMATCH at currItem %u, !OPTIONAL, abort\n", i); + return DR_UnexpectedTag; + } + + /* else this was optional, on to next item */ + } /* searching for tag match */ + + if(foundMatch == 0) { + /* + * Found an item we couldn't match to any tag spec and we're at + * the end. + */ + derDecDbg("--- TAG NOT FOUND, abort\n"); + return DR_UnexpectedTag; + } + + /* else on to next item */ + } /* main loop */ + + /* + * If we get here, there appears to be more to process, but we've + * given the caller everything they want. + */ + return DR_Success; + } +} +#endif + +#endif /* DER_DECODE_ENABLE */ diff --git a/view/kernelcache/core/transformers/libDER/DER_Decode.h b/view/kernelcache/core/transformers/libDER/DER_Decode.h new file mode 100644 index 00000000..403458b4 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/DER_Decode.h @@ -0,0 +1,206 @@ +/* + * Copyright (c) 2005-2011 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + +/* + * DER_Decode.h - DER decoding routines + */ +/* + * NOTICE: This file was modified by xerub to reflect binary code. + */ + +#ifndef _DER_DECODE_H_ +#define _DER_DECODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <libDER/libDER.h> +#include <stdbool.h> + +/* + * Decoding one item consists of extracting its tag, a pointer + * to the actual content, and the length of the content. Those + * three are represented by a DERDecodedInfo. + */ +typedef struct { + DERTag tag; + DERItem content; +} DERDecodedInfo; + +/* + * Basic decoding primitive. Only works with: + * + * -- definite length encoding + * -- one-byte tags + * -- max content length fits in a DERSize + * + * No malloc or copy of the contents is performed; the returned + * content->content.data is a pointer into the incoming der data. + */ +DERReturn DERDecodeItem( + const DERItem *der, /* data to decode */ + DERDecodedInfo *decoded); /* RETURNED */ + +DERReturn DERDecodeItemPartialBuffer( + const DERItem *der, /* data to decode */ + DERDecodedInfo *decoded, /* RETURNED */ + bool partial); + +/* + * Given a BIT_STRING, in the form of its raw content bytes, + * obtain the number of unused bits and the raw bit string bytes. + */ +DERReturn DERParseBitString( + const DERItem *contents, + DERItem *bitStringBytes, /* RETURNED */ + DERByte *numUnusedBits); /* RETURNED */ + +/* + * Given a BOOLEAN, in the form of its raw content bytes, + * obtain it's value. + */ +DERReturn DERParseBoolean( + const DERItem *contents, + bool *value); /* RETURNED */ + +DERReturn DERParseInteger( + const DERItem *contents, + uint32_t *value); /* RETURNED */ + +DERReturn DERParseInteger64( + const DERItem *contents, + uint64_t *value); /* RETURNED */ + +/* + * Sequence/set decode support. + */ + +/* state representing a sequence or set being decoded */ +typedef struct { + DERByte *nextItem; + DERByte *end; +} DERSequence; + +/* + * To decode a set or sequence, call DERDecodeSeqInit or + * DERDecodeSeqContentInit once, then call DERDecodeSeqNext to + * get each enclosed item. + * + * DERDecodeSeqNext returns DR_EndOfSequence when no more + * items are available. + */ + +/* + * Use this to parse the top level sequence's tag and content length. + */ +DERReturn DERDecodeSeqInit( + const DERItem *der, /* data to decode */ + DERTag *tag, /* RETURNED tag of sequence/set. This will be + * either ASN1_CONSTR_SEQUENCE or + * ASN1_CONSTR_SET. */ + DERSequence *derSeq); /* RETURNED, to use in DERDecodeSeqNext */ + +/* + * Use this to start in on decoding a sequence's content, when + * the top-level tag and content have already been decoded. + */ +DERReturn DERDecodeSeqContentInit( + const DERItem *content, + DERSequence *derSeq); /* RETURNED, to use in DERDecodeSeqNext */ + +/* obtain the next decoded item in a sequence or set */ +DERReturn DERDecodeSeqNext( + DERSequence *derSeq, + DERDecodedInfo *decoded); /* RETURNED */ + +/* + * High level sequence decode. + */ + +/* + * Per-item decode options. + */ + +/* Explicit default, no options */ +#define DER_DEC_NO_OPTS 0x0000 + +/* This item optional, can be skipped during decode */ +#define DER_DEC_OPTIONAL 0x0001 + +/* Skip the tag check; accept anything. */ +#define DER_DEC_ASN_ANY 0x0002 + +/* Skip item, no write to DERDecodedInfo (but tag check still performed) */ +#define DER_DEC_SKIP 0x0004 + +/* Save full DER encoding in DERDecodedInfo, including tag and length. Normally + * only the content is saved. */ +#define DER_DEC_SAVE_DER 0x0008 + +/* + * High level sequence parse, starting with top-level tag and content. + * Top level tag must be ASN1_CONSTR_SEQUENCE - if it's not, and that's + * OK, use DERParseSequenceContent(). + * + * These never return DR_EndOfSequence - if an *unexpected* end of sequence + * occurs, return DR_IncompleteSeq. + * + * Results of the decoding of one item are placed in a DERItem whose address + * is the dest arg plus the offset value in the associated DERItemSpec. + * + * Items which are optional (DER_DEC_OPTIONAL) and which are not found, + * leave their associated DERDecodedInfos unmodified. + * + * Processing of a sequence ends on detection of any error or after the + * last DERItemSpec is processed. + * + * The sizeToZero argument, if nonzero, indicates the number of bytes + * starting at dest to zero before processing the sequence. This is + * generally desirable, particularly if there are any DER_DEC_OPTIONAL + * items in the sequence; skipped optional items are detected by the + * caller via a NULL DERDecodedInfo.content.data; if this hasn't been + * explicitly zeroed (generally, by passing a nonzero value of sizeToZero), + * skipped items can't be detected. + */ +DERReturn DERParseSequence( + const DERItem *der, + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + void *dest, /* DERDecodedInfo(s) here RETURNED */ + DERSize sizeToZero); /* optional */ + +/* high level sequence parse, starting with sequence's content */ +DERReturn DERParseSequenceContent( + const DERItem *content, + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + void *dest, /* DERDecodedInfo(s) here RETURNED */ + DERSize sizeToZero); /* optional */ + +#ifdef __cplusplus +} +#endif + +#endif /* _DER_DECODE_H_ */ + diff --git a/view/kernelcache/core/transformers/libDER/DER_Encode.c b/view/kernelcache/core/transformers/libDER/DER_Encode.c new file mode 100644 index 00000000..9b694617 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/DER_Encode.c @@ -0,0 +1,365 @@ +/* + * Copyright (c) 2005-2007,2011,2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * DER_Encode.h - DER encoding routines + * + */ + +#include <libDER/DER_Encode.h> +#include <libDER/asn1Types.h> +#include <libDER/libDER_config.h> +#include <libDER/DER_Decode.h> + +#ifndef DER_ENCODE_ENABLE +#error Please define DER_ENCODE_ENABLE. +#endif + +#if DER_ENCODE_ENABLE + +/* calculate size of encoded tag */ +static DERSize DERLengthOfTag( + DERTag tag) +{ + DERSize rtn = 1; + + tag &= ASN1_TAGNUM_MASK; + if (tag >= 0x1F) { + /* Shift 7-bit digits out of the tag integer until it's zero. */ + while(tag != 0) { + rtn++; + tag >>= 7; + } + } + + return rtn; +} + +/* encode tag */ +static DERReturn DEREncodeTag( + DERTag tag, + DERByte *buf, /* encoded length goes here */ + DERSize *inOutLen) /* IN/OUT */ +{ + DERSize outLen = DERLengthOfTag(tag); + DERTag tagNumber = tag & ASN1_TAGNUM_MASK; + DERByte tag1 = (tag >> (sizeof(DERTag) * 8 - 8)) & 0xE0; + + if(outLen > *inOutLen) { + return DR_BufOverflow; + } + + if(outLen == 1) { + /* short form */ + *buf = tag1 | tagNumber; + } + else { + /* long form */ + DERByte *tagBytes = buf + outLen; // l.s. digit of tag + *buf = tag1 | 0x1F; // tag class / method indicator + *--tagBytes = tagNumber & 0x7F; + tagNumber >>= 7; + while(tagNumber != 0) { + *--tagBytes = (tagNumber & 0x7F) | 0x80; + tagNumber >>= 7; + } + } + *inOutLen = outLen; + return DR_Success; +} + +/* calculate size of encoded length */ +DERSize DERLengthOfLength( + DERSize length) +{ + DERSize rtn; + + if(length < 0x80) { + /* short form length */ + return 1; + } + + /* long form - one length-of-length byte plus length bytes */ + rtn = 1; + while(length != 0) { + rtn++; + length >>= 8; + } + return rtn; +} + +/* encode length */ +DERReturn DEREncodeLength( + DERSize length, + DERByte *buf, /* encoded length goes here */ + DERSize *inOutLen) /* IN/OUT */ +{ + DERByte *lenBytes; + DERSize outLen = DERLengthOfLength(length); + + if(outLen > *inOutLen) { + return DR_BufOverflow; + } + + if(length < 0x80) { + /* short form */ + *buf = (DERByte)length; + *inOutLen = 1; + return DR_Success; + } + + /* long form */ + *buf = (outLen - 1) | 0x80; // length of length, long form indicator + lenBytes = buf + outLen - 1; // l.s. digit of length + while(length != 0) { + *lenBytes-- = (DERByte)length; + length >>= 8; + } + *inOutLen = outLen; + return DR_Success; +} + +DERSize DERLengthOfItem( + DERTag tag, + DERSize length) +{ + return DERLengthOfTag(tag) + DERLengthOfLength(length) + length; +} + +DERReturn DEREncodeItem( + DERTag tag, + DERSize length, + const DERByte *src, + DERByte *derOut, /* encoded item goes here */ + DERSize *inOutLen) /* IN/OUT */ +{ + DERReturn drtn; + DERSize itemLen; + DERByte *currPtr = derOut; + DERSize bytesLeft = DERLengthOfItem(tag, length); + if(bytesLeft > *inOutLen) { + return DR_BufOverflow; + } + *inOutLen = bytesLeft; + + /* top level tag */ + itemLen = bytesLeft; + drtn = DEREncodeTag(tag, currPtr, &itemLen); + if(drtn) { + return drtn; + } + currPtr += itemLen; + bytesLeft -= itemLen; + itemLen = bytesLeft; + drtn = DEREncodeLength(length, currPtr, &itemLen); + if(drtn) { + return drtn; + } + currPtr += itemLen; + bytesLeft -= itemLen; + DERMemmove(currPtr, src, length); + + (void) bytesLeft; + + return DR_Success; +} + +static /* calculate the content length of an encoded sequence */ +DERSize DERContentLengthOfEncodedSequence( + const void *src, /* generally a ptr to a struct full of + * DERItems */ + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs) +{ + DERSize contentLen = 0; + unsigned dex; + DERSize thisContentLen; + + /* find length of each item */ + for(dex=0; dex<numItems; dex++) { + const DERItemSpec *currItemSpec = &itemSpecs[dex]; + DERShort currOptions = currItemSpec->options; + const DERByte *byteSrc = (const DERByte *)src + currItemSpec->offset; + const DERItem *itemSrc = (const DERItem *)byteSrc; + + if(currOptions & DER_ENC_WRITE_DER) { + /* easy case - no encode */ + contentLen += itemSrc->length; + continue; + } + + if ((currOptions & DER_DEC_OPTIONAL) && itemSrc->length == 0) { + /* If an optional item isn't present we don't encode a + tag and len. */ + continue; + } + + /* + * length of this item = + * tag (one byte) + + * length of length + + * content length + + * optional zero byte for signed integer + */ + contentLen += DERLengthOfTag(currItemSpec->tag); + + /* check need for pad byte before calculating lengthOfLength... */ + thisContentLen = itemSrc->length; + if((currOptions & DER_ENC_SIGNED_INT) && + (itemSrc->length != 0)) { + if(itemSrc->data[0] & 0x80) { + /* insert zero keep it positive */ + thisContentLen++; + } + } + contentLen += DERLengthOfLength(thisContentLen); + contentLen += thisContentLen; + } + return contentLen; +} + +DERReturn DEREncodeSequence( + DERTag topTag, /* ASN1_CONSTR_SEQUENCE, ASN1_CONSTR_SET */ + const void *src, /* generally a ptr to a struct full of + * DERItems */ + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + DERByte *derOut, /* encoded data written here */ + DERSize *inOutLen) /* IN/OUT */ +{ + const DERByte *endPtr = derOut + *inOutLen; + DERByte *currPtr = derOut; + DERSize bytesLeft = *inOutLen; + DERSize contentLen; + DERReturn drtn; + DERSize itemLen; + unsigned dex; + + /* top level tag */ + itemLen = bytesLeft; + drtn = DEREncodeTag(topTag, currPtr, &itemLen); + if(drtn) { + return drtn; + } + currPtr += itemLen; + bytesLeft -= itemLen; + if(currPtr >= endPtr) { + return DR_BufOverflow; + } + + /* content length */ + contentLen = DERContentLengthOfEncodedSequence(src, numItems, itemSpecs); + itemLen = bytesLeft; + drtn = DEREncodeLength(contentLen, currPtr, &itemLen); + if(drtn) { + return drtn; + } + currPtr += itemLen; + bytesLeft -= itemLen; + if(currPtr + contentLen > endPtr) { + return DR_BufOverflow; + } + /* we don't have to check for overflow any more */ + + /* grind thru the items */ + for(dex=0; dex<numItems; dex++) { + const DERItemSpec *currItemSpec = &itemSpecs[dex]; + DERShort currOptions = currItemSpec->options; + const DERByte *byteSrc = (const DERByte *)src + currItemSpec->offset; + const DERItem *itemSrc = (const DERItem *)byteSrc; + int prependZero = 0; + + if(currOptions & DER_ENC_WRITE_DER) { + /* easy case */ + DERMemmove(currPtr, itemSrc->data, itemSrc->length); + currPtr += itemSrc->length; + bytesLeft -= itemSrc->length; + continue; + } + + if ((currOptions & DER_DEC_OPTIONAL) && itemSrc->length == 0) { + /* If an optional item isn't present we skip it. */ + continue; + } + + /* encode one item: first the tag */ + itemLen = bytesLeft; + drtn = DEREncodeTag(currItemSpec->tag, currPtr, &itemLen); + if(drtn) { + return drtn; + } + currPtr += itemLen; + bytesLeft -= itemLen; + + /* do we need to prepend a zero to content? */ + contentLen = itemSrc->length; + if((currOptions & DER_ENC_SIGNED_INT) && + (itemSrc->length != 0)) { + if(itemSrc->data[0] & 0x80) { + /* insert zero keep it positive */ + contentLen++; + prependZero = 1; + } + } + + /* encode content length */ + itemLen = bytesLeft; + drtn = DEREncodeLength(contentLen, currPtr, &itemLen); + if(drtn) { + return drtn; + } + currPtr += itemLen; + bytesLeft -= itemLen; + + /* now the content, with possible leading zero added */ + if(prependZero) { + *currPtr++ = 0; + bytesLeft--; + } + DERMemmove(currPtr, itemSrc->data, itemSrc->length); + currPtr += itemSrc->length; + bytesLeft -= itemSrc->length; + } + *inOutLen = (currPtr - derOut); + return DR_Success; +} + +/* calculate the length of an encoded sequence. */ +DERSize DERLengthOfEncodedSequence( + DERTag topTag, + const void *src, /* generally a ptr to a struct full of + * DERItems */ + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs) +{ + DERSize contentLen = DERContentLengthOfEncodedSequence( + src, numItems, itemSpecs); + + return DERLengthOfTag(topTag) + + DERLengthOfLength(contentLen) + + contentLen; +} + +#endif /* DER_ENCODE_ENABLE */ + diff --git a/view/kernelcache/core/transformers/libDER/DER_Encode.h b/view/kernelcache/core/transformers/libDER/DER_Encode.h new file mode 100644 index 00000000..6dc3d636 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/DER_Encode.h @@ -0,0 +1,124 @@ +/* + * Copyright (c) 2005-2007,2011,2013-2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * DER_Encode.h - DER encoding routines + * + */ + +#ifndef _DER_ENCODE_H_ +#define _DER_ENCODE_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <libDER/libDER.h> + +/* + * Max size of an encoded item given its length. + * This includes a possible leading zero prepended to a signed integer + * (see DER_ENC_SIGNED_INT below). + */ +#define DER_MAX_ENCODED_SIZE(len) \ + ( 1 + /* tag */ \ + 5 + /* max length */ \ + 1 + /* possible prepended zero */ \ + len) + +/* calculate size of encoded length */ +DERSize DERLengthOfLength( + DERSize length); + +/* encode length */ +DERReturn DEREncodeLength( + DERSize length, + DERByte *buf, /* encoded length goes here */ + DERSize *inOutLen); /* IN/OUT */ + +/* calculate size of encoded length */ +DERSize DERLengthOfItem( + DERTag tag, + DERSize length); + +/* encode item */ +DERReturn DEREncodeItem( + DERTag tag, + DERSize length, + const DERByte *src, + DERByte *derOut, /* encoded item goes here */ + DERSize *inOutLen); /* IN/OUT */ + +/* + * Per-item encode options. + */ + +/* explicit default, no options */ +#define DER_ENC_NO_OPTS 0x0000 + +/* signed integer check: if incoming m.s. bit is 1, prepend a zero */ +#define DER_ENC_SIGNED_INT 0x0100 + +/* DERItem contains fully encoded item - copy, don't encode */ +#define DER_ENC_WRITE_DER 0x0200 + + +/* + * High-level sequence or set encode support. + * + * The outgoing sequence is expressed as an array of DERItemSpecs, each + * of which corresponds to one item in the encoded sequence. + * + * Normally the tag of the encoded item comes from the associated + * DERItemSpec, and the content comes from the DERItem whose address is + * the src arg plus the offset value in the associated DERItemSpec. + * + * If the DER_ENC_WRITE_DER option is true for a given DERItemSpec then + * no per-item encoding is done; the DER - with tag, length, and content - + * is taken en masse from the associated DERItem. + */ +DERReturn DEREncodeSequence( + DERTag topTag, /* ASN1_CONSTR_SEQUENCE, ASN1_CONSTR_SET */ + const void *src, /* generally a ptr to a struct full of + * DERItems */ + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs, + DERByte *derOut, /* encoded data written here */ + DERSize *inOutLen); /* IN/OUT */ + +/* precalculate the length of an encoded sequence. */ +DERSize DERLengthOfEncodedSequence( + DERTag topTag, /* ASN1_CONSTR_SEQUENCE, ASN1_CONSTR_SET */ + const void *src, /* generally a ptr to a struct full of + * DERItems */ + DERShort numItems, /* size of itemSpecs[] */ + const DERItemSpec *itemSpecs); + + +#ifdef __cplusplus +} +#endif + +#endif /* _DER_ENCODE_H_ */ + diff --git a/view/kernelcache/core/transformers/libDER/LICENSE b/view/kernelcache/core/transformers/libDER/LICENSE new file mode 100644 index 00000000..2116b0a8 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/LICENSE @@ -0,0 +1,60 @@ +APPLE PUBLIC SOURCE LICENSE +Version 2.0 - August 6, 2003 + +Please read this License carefully before downloading this software. By downloading or using this software, you are agreeing to be bound by the terms of this License. If you do not or cannot agree to the terms of this License, please do not download or use the software. + +Apple Note: In January 2007, Apple changed its corporate name from "Apple Computer, Inc." to "Apple Inc." This change has been reflected below and copyright years updated, but no other changes have been made to the APSL 2.0. + + 1. General; Definitions. This License applies to any program or other work which Apple Inc. ("Apple") makes publicly available and which contains a notice placed by Apple identifying such program or work as "Original Code" and stating that it is subject to the terms of this Apple Public Source License version 2.0 ("License"). As used in this License: + 1.1 "Applicable Patent Rights" mean: (a) in the case where Apple is the grantor of rights, (i) claims of patents that are now or hereafter acquired, owned by or assigned to Apple and (ii) that cover subject matter contained in the Original Code, but only to the extent necessary to use, reproduce and/or distribute the Original Code without infringement; and (b) in the case where You are the grantor of rights, (i) claims of patents that are now or hereafter acquired, owned by or assigned to You and (ii) that cover subject matter in Your Modifications, taken alone or in combination with Original Code. + 1.2 "Contributor" means any person or entity that creates or contributes to the creation of Modifications. + 1.3 "Covered Code" means the Original Code, Modifications, the combination of Original Code and any Modifications, and/or any respective portions thereof. + 1.4 "Externally Deploy" means: (a) to sublicense, distribute or otherwise make Covered Code available, directly or indirectly, to anyone other than You; and/or (b) to use Covered Code, alone or as part of a Larger Work, in any way to provide a service, including but not limited to delivery of content, through electronic communication with a client other than You. + 1.5 "Larger Work" means a work which combines Covered Code or portions thereof with code not governed by the terms of this License. + 1.6 "Modifications" mean any addition to, deletion from, and/or change to, the substance and/or structure of the Original Code, any previous Modifications, the combination of Original Code and any previous Modifications, and/or any respective portions thereof. When code is released as a series of files, a Modification is: (a) any addition to or deletion from the contents of a file containing Covered Code; and/or (b) any new file or other representation of computer program statements that contains any part of Covered Code. + 1.7 "Original Code" means (a) the Source Code of a program or other work as originally made available by Apple under this License, including the Source Code of any updates or upgrades to such programs or works made available by Apple under this License, and that has been expressly identified by Apple as such in the header file(s) of such work; and (b) the object code compiled from such Source Code and originally made available by Apple under this License + 1.8 "Source Code" means the human readable form of a program or other work that is suitable for making modifications to it, including all modules it contains, plus any associated interface definition files, scripts used to control compilation and installation of an executable (object code). + 1.9 "You" or "Your" means an individual or a legal entity exercising rights under this License. For legal entities, "You" or "Your" includes any entity which controls, is controlled by, or is under common control with, You, where "control" means (a) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (b) ownership of fifty percent (50%) or more of the outstanding shares or beneficial ownership of such entity. + 2. Permitted Uses; Conditions & Restrictions. Subject to the terms and conditions of this License, Apple hereby grants You, effective on the date You accept this License and download the Original Code, a world-wide, royalty-free, non-exclusive license, to the extent of Apple's Applicable Patent Rights and copyrights covering the Original Code, to do the following: + 2.1 Unmodified Code. You may use, reproduce, display, perform, internally distribute within Your organization, and Externally Deploy verbatim, unmodified copies of the Original Code, for commercial or non-commercial purposes, provided that in each instance: + (a) You must retain and reproduce in all copies of Original Code the copyright and other proprietary notices and disclaimers of Apple as they appear in the Original Code, and keep intact all notices in the Original Code that refer to this License; and + (b) You must include a copy of this License with every copy of Source Code of Covered Code and documentation You distribute or Externally Deploy, and You may not offer or impose any terms on such Source Code that alter or restrict this License or the recipients' rights hereunder, except as permitted under Section 6. + 2.2 Modified Code. You may modify Covered Code and use, reproduce, display, perform, internally distribute within Your organization, and Externally Deploy Your Modifications and Covered Code, for commercial or non-commercial purposes, provided that in each instance You also meet all of these conditions: + (a) You must satisfy all the conditions of Section 2.1 with respect to the Source Code of the Covered Code; + (b) You must duplicate, to the extent it does not already exist, the notice in Exhibit A in each file of the Source Code of all Your Modifications, and cause the modified files to carry prominent notices stating that You changed the files and the date of any change; and + (c) If You Externally Deploy Your Modifications, You must make Source Code of all Your Externally Deployed Modifications either available to those to whom You have Externally Deployed Your Modifications, or publicly available. Source Code of Your Externally Deployed Modifications must be released under the terms set forth in this License, including the license grants set forth in Section 3 below, for as long as you Externally Deploy the Covered Code or twelve (12) months from the date of initial External Deployment, whichever is longer. You should preferably distribute the Source Code of Your Externally Deployed Modifications electronically (e.g. download from a web site). + 2.3 Distribution of Executable Versions. In addition, if You Externally Deploy Covered Code (Original Code and/or Modifications) in object code, executable form only, You must include a prominent notice, in the code itself as well as in related documentation, stating that Source Code of the Covered Code is available under the terms of this License with information on how and where to obtain such Source Code. + 2.4 Third Party Rights. You expressly acknowledge and agree that although Apple and each Contributor grants the licenses to their respective portions of the Covered Code set forth herein, no assurances are provided by Apple or any Contributor that the Covered Code does not infringe the patent or other intellectual property rights of any other entity. Apple and each Contributor disclaim any liability to You for claims brought by any other entity based on infringement of intellectual property rights or otherwise. As a condition to exercising the rights and licenses granted hereunder, You hereby assume sole responsibility to secure any other intellectual property rights needed, if any. For example, if a third party patent license is required to allow You to distribute the Covered Code, it is Your responsibility to acquire that license before distributing the Covered Code. + 3. Your Grants. In consideration of, and as a condition to, the licenses granted to You under this License, You hereby grant to any person or entity receiving or distributing Covered Code under this License a non-exclusive, royalty-free, perpetual, irrevocable license, under Your Applicable Patent Rights and other intellectual property rights (other than patent) owned or controlled by You, to use, reproduce, display, perform, modify, sublicense, distribute and Externally Deploy Your Modifications of the same scope and extent as Apple's licenses under Sections 2.1 and 2.2 above. + 4. Larger Works. You may create a Larger Work by combining Covered Code with other code not governed by the terms of this License and distribute the Larger Work as a single product. In each such instance, You must make sure the requirements of this License are fulfilled for the Covered Code or any portion thereof. + 5. Limitations on Patent License. Except as expressly stated in Section 2, no other patent rights, express or implied, are granted by Apple herein. Modifications and/or Larger Works may require additional patent licenses from Apple which Apple may grant in its sole discretion. + 6. Additional Terms. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations and/or other rights consistent with the scope of the license granted herein ("Additional Terms") to one or more recipients of Covered Code. However, You may do so only on Your own behalf and as Your sole responsibility, and not on behalf of Apple or any Contributor. You must obtain the recipient's agreement that any such Additional Terms are offered by You alone, and You hereby agree to indemnify, defend and hold Apple and every Contributor harmless for any liability incurred by or claims asserted against Apple or such Contributor by reason of any such Additional Terms. + 7. Versions of the License. Apple may publish revised and/or new versions of this License from time to time. Each version will be given a distinguishing version number. Once Original Code has been published under a particular version of this License, You may continue to use it under the terms of that version. You may also choose to use such Original Code under the terms of any subsequent version of this License published by Apple. No one other than Apple has the right to modify the terms applicable to Covered Code created under this License. + 8. NO WARRANTY OR SUPPORT. The Covered Code may contain in whole or in part pre-release, untested, or not fully tested works. The Covered Code may contain errors that could cause failures or loss of data, and may be incomplete or contain inaccuracies. You expressly acknowledge and agree that use of the Covered Code, or any portion thereof, is at Your sole and entire risk. THE COVERED CODE IS PROVIDED "AS IS" AND WITHOUT WARRANTY, UPGRADES OR SUPPORT OF ANY KIND AND APPLE AND APPLE'S LICENSOR(S) (COLLECTIVELY REFERRED TO AS "APPLE" FOR THE PURPOSES OF SECTIONS 8 AND 9) AND ALL CONTRIBUTORS EXPRESSLY DISCLAIM ALL WARRANTIES AND/OR CONDITIONS, EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES AND/OR CONDITIONS OF MERCHANTABILITY, OF SATISFACTORY QUALITY, OF FITNESS FOR A PARTICULAR PURPOSE, OF ACCURACY, OF QUIET ENJOYMENT, AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. APPLE AND EACH CONTRIBUTOR DOES NOT WARRANT AGAINST INTERFERENCE WITH YOUR ENJOYMENT OF THE COVERED CODE, THAT THE FUNCTIONS CONTAINED IN THE COVERED CODE WILL MEET YOUR REQUIREMENTS, THAT THE OPERATION OF THE COVERED CODE WILL BE UNINTERRUPTED OR ERROR-FREE, OR THAT DEFECTS IN THE COVERED CODE WILL BE CORRECTED. NO ORAL OR WRITTEN INFORMATION OR ADVICE GIVEN BY APPLE, AN APPLE AUTHORIZED REPRESENTATIVE OR ANY CONTRIBUTOR SHALL CREATE A WARRANTY. You acknowledge that the Covered Code is not intended for use in the operation of nuclear facilities, aircraft navigation, communication systems, or air traffic control machines in which case the failure of the Covered Code could lead to death, personal injury, or severe physical or environmental damage. + 9. LIMITATION OF LIABILITY. TO THE EXTENT NOT PROHIBITED BY LAW, IN NO EVENT SHALL APPLE OR ANY CONTRIBUTOR BE LIABLE FOR ANY INCIDENTAL, SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES ARISING OUT OF OR RELATING TO THIS LICENSE OR YOUR USE OR INABILITY TO USE THE COVERED CODE, OR ANY PORTION THEREOF, WHETHER UNDER A THEORY OF CONTRACT, WARRANTY, TORT (INCLUDING NEGLIGENCE), PRODUCTS LIABILITY OR OTHERWISE, EVEN IF APPLE OR SUCH CONTRIBUTOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES AND NOTWITHSTANDING THE FAILURE OF ESSENTIAL PURPOSE OF ANY REMEDY. SOME JURISDICTIONS DO NOT ALLOW THE LIMITATION OF LIABILITY OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO THIS LIMITATION MAY NOT APPLY TO YOU. In no event shall Apple's total liability to You for all damages (other than as may be required by applicable law) under this License exceed the amount of fifty dollars ($50.00). + 10. Trademarks. This License does not grant any rights to use the trademarks or trade names "Apple", "Mac", "Mac OS", "QuickTime", "QuickTime Streaming Server" or any other trademarks, service marks, logos or trade names belonging to Apple (collectively "Apple Marks") or to any trademark, service mark, logo or trade name belonging to any Contributor. You agree not to use any Apple Marks in or as part of the name of products derived from the Original Code or to endorse or promote products derived from the Original Code other than as expressly permitted by and in strict compliance at all times with Apple's third party trademark usage guidelines which are posted at http://www.apple.com/legal/guidelinesfor3rdparties.html. + 11. Ownership. Subject to the licenses granted under this License, each Contributor retains all rights, title and interest in and to any Modifications made by such Contributor. Apple retains all rights, title and interest in and to the Original Code and any Modifications made by or on behalf of Apple ("Apple Modifications"), and such Apple Modifications will not be automatically subject to this License. Apple may, at its sole discretion, choose to license such Apple Modifications under this License, or on different terms from those contained in this License or may choose not to license them at all. + 12. Termination. + 12.1 Termination. This License and the rights granted hereunder will terminate: + (a) automatically without notice from Apple if You fail to comply with any term(s) of this License and fail to cure such breach within 30 days of becoming aware of such breach; + (b) immediately in the event of the circumstances described in Section 13.5(b); or + (c) automatically without notice from Apple if You, at any time during the term of this License, commence an action for patent infringement against Apple; provided that Apple did not first commence an action for patent infringement against You in that instance. + 12.2 Effect of Termination. Upon termination, You agree to immediately stop any further use, reproduction, modification, sublicensing and distribution of the Covered Code. All sublicenses to the Covered Code which have been properly granted prior to termination shall survive any termination of this License. Provisions which, by their nature, should remain in effect beyond the termination of this License shall survive, including but not limited to Sections 3, 5, 8, 9, 10, 11, 12.2 and 13. No party will be liable to any other for compensation, indemnity or damages of any sort solely as a result of terminating this License in accordance with its terms, and termination of this License will be without prejudice to any other right or remedy of any party. + 13. Miscellaneous. + 13.1 Government End Users. The Covered Code is a "commercial item" as defined in FAR 2.101. Government software and technical data rights in the Covered Code include only those rights customarily provided to the public as defined in this License. This customary commercial license in technical data and software is provided in accordance with FAR 12.211 (Technical Data) and 12.212 (Computer Software) and, for Department of Defense purchases, DFAR 252.227-7015 (Technical Data -- Commercial Items) and 227.7202-3 (Rights in Commercial Computer Software or Computer Software Documentation). Accordingly, all U.S. Government End Users acquire Covered Code with only those rights set forth herein. + 13.2 Relationship of Parties. This License will not be construed as creating an agency, partnership, joint venture or any other form of legal association between or among You, Apple or any Contributor, and You will not represent to the contrary, whether expressly, by implication, appearance or otherwise. + 13.3 Independent Development. Nothing in this License will impair Apple's right to acquire, license, develop, have others develop for it, market and/or distribute technology or products that perform the same or similar functions as, or otherwise compete with, Modifications, Larger Works, technology or products that You may develop, produce, market or distribute. + 13.4 Waiver; Construction. Failure by Apple or any Contributor to enforce any provision of this License will not be deemed a waiver of future enforcement of that or any other provision. Any law or regulation which provides that the language of a contract shall be construed against the drafter will not apply to this License. + 13.5 Severability. (a) If for any reason a court of competent jurisdiction finds any provision of this License, or portion thereof, to be unenforceable, that provision of the License will be enforced to the maximum extent permissible so as to effect the economic benefits and intent of the parties, and the remainder of this License will continue in full force and effect. (b) Notwithstanding the foregoing, if applicable law prohibits or restricts You from fully and/or specifically complying with Sections 2 and/or 3 or prevents the enforceability of either of those Sections, this License will immediately terminate and You must immediately discontinue any use of the Covered Code and destroy all copies of it that are in your possession or control. + 13.6 Dispute Resolution. Any litigation or other dispute resolution between You and Apple relating to this License shall take place in the Northern District of California, and You and Apple hereby consent to the personal jurisdiction of, and venue in, the state and federal courts within that District with respect to this License. The application of the United Nations Convention on Contracts for the International Sale of Goods is expressly excluded. + 13.7 Entire Agreement; Governing Law. This License constitutes the entire agreement between the parties with respect to the subject matter hereof. This License shall be governed by the laws of the United States and the State of California, except that body of California law concerning conflicts of law. + + Where You are located in the province of Quebec, Canada, the following clause applies: The parties hereby confirm that they have requested that this License and all related documents be drafted in English. Les parties ont exigé que le présent contrat et tous les documents connexes soient rédigés en anglais. + +EXHIBIT A. + +"Portions Copyright (c) 1999-2007 Apple Inc. All Rights Reserved. + +This file contains Original Code and/or Modifications of Original Code as defined in and that are subject to the Apple Public Source License Version 2.0 (the 'License'). You may not use this file except in compliance with the License. Please obtain a copy of the License at http://www.opensource.apple.com/apsl/ and read it before using this file. + +The Original Code and all software distributed under the License are distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. Please see the License for the specific language governing rights and limitations under the License." diff --git a/view/kernelcache/core/transformers/libDER/asn1Types.h b/view/kernelcache/core/transformers/libDER/asn1Types.h new file mode 100644 index 00000000..5275af3c --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/asn1Types.h @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2005-2007,2011,2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * asn1Types.h - ASN.1/DER #defines - strictly hard coded per the real world + * + */ + +#ifndef _ASN1_TYPES_H_ +#define _ASN1_TYPES_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +/* copied from libsecurity_asn1 project */ + +#define ASN1_BOOLEAN 0x01 +#define ASN1_INTEGER 0x02 +#define ASN1_BIT_STRING 0x03 +#define ASN1_OCTET_STRING 0x04 +#define ASN1_NULL 0x05 +#define ASN1_OBJECT_ID 0x06 +#define ASN1_OBJECT_DESCRIPTOR 0x07 +/* External type and instance-of type 0x08 */ +#define ASN1_REAL 0x09 +#define ASN1_ENUMERATED 0x0a +#define ASN1_EMBEDDED_PDV 0x0b +#define ASN1_UTF8_STRING 0x0c +/* 0x0d */ +/* 0x0e */ +/* 0x0f */ +#define ASN1_SEQUENCE 0x10 +#define ASN1_SET 0x11 +#define ASN1_NUMERIC_STRING 0x12 +#define ASN1_PRINTABLE_STRING 0x13 +#define ASN1_T61_STRING 0x14 +#define ASN1_VIDEOTEX_STRING 0x15 +#define ASN1_IA5_STRING 0x16 +#define ASN1_UTC_TIME 0x17 +#define ASN1_GENERALIZED_TIME 0x18 +#define ASN1_GRAPHIC_STRING 0x19 +#define ASN1_VISIBLE_STRING 0x1a +#define ASN1_GENERAL_STRING 0x1b +#define ASN1_UNIVERSAL_STRING 0x1c +/* 0x1d */ +#define ASN1_BMP_STRING 0x1e +#define ASN1_HIGH_TAG_NUMBER 0x1f +#define ASN1_TELETEX_STRING ASN1_T61_STRING + +#ifdef DER_MULTIBYTE_TAGS + +#define ASN1_TAG_MASK ((DERTag)~0) +#define ASN1_TAGNUM_MASK ((DERTag)~((DERTag)7 << (sizeof(DERTag) * 8 - 3))) + +#define ASN1_METHOD_MASK ((DERTag)1 << (sizeof(DERTag) * 8 - 3)) +#define ASN1_PRIMITIVE ((DERTag)0 << (sizeof(DERTag) * 8 - 3)) +#define ASN1_CONSTRUCTED ((DERTag)1 << (sizeof(DERTag) * 8 - 3)) + +#define ASN1_CLASS_MASK ((DERTag)3 << (sizeof(DERTag) * 8 - 2)) +#define ASN1_UNIVERSAL ((DERTag)0 << (sizeof(DERTag) * 8 - 2)) +#define ASN1_APPLICATION ((DERTag)1 << (sizeof(DERTag) * 8 - 2)) +#define ASN1_CONTEXT_SPECIFIC ((DERTag)2 << (sizeof(DERTag) * 8 - 2)) +#define ASN1_PRIVATE ((DERTag)3 << (sizeof(DERTag) * 8 - 2)) + +#else /* DER_MULTIBYTE_TAGS */ + +#define ASN1_TAG_MASK 0xff +#define ASN1_TAGNUM_MASK 0x1f +#define ASN1_METHOD_MASK 0x20 +#define ASN1_PRIMITIVE 0x00 +#define ASN1_CONSTRUCTED 0x20 + +#define ASN1_CLASS_MASK 0xc0 +#define ASN1_UNIVERSAL 0x00 +#define ASN1_APPLICATION 0x40 +#define ASN1_CONTEXT_SPECIFIC 0x80 +#define ASN1_PRIVATE 0xc0 + +#endif /* !DER_MULTIBYTE_TAGS */ + +/* sequence and set appear as the following */ +#define ASN1_CONSTR_SEQUENCE (ASN1_CONSTRUCTED | ASN1_SEQUENCE) +#define ASN1_CONSTR_SET (ASN1_CONSTRUCTED | ASN1_SET) + +#ifdef __cplusplus +} +#endif + +#endif /* _ASN1_TYPES_H_ */ + diff --git a/view/kernelcache/core/transformers/libDER/libDER.h b/view/kernelcache/core/transformers/libDER/libDER.h new file mode 100644 index 00000000..cde53fe5 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/libDER.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2005-2007,2011,2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * libDER.h - main header for libDER, a ROM-capable DER decoding library. + * + */ + +#ifndef _LIB_DER_H_ +#define _LIB_DER_H_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include <libDER/libDER_config.h> +/* + * Error returns generated by this library. + */ +typedef enum { + DR_Success, + DR_EndOfSequence, /* end of sequence or set */ + DR_UnexpectedTag, /* unexpected tag found while decoding */ + DR_DecodeError, /* misc. decoding error (badly formatted DER) */ + DR_Unimplemented, /* function not implemented in this configuration */ + DR_IncompleteSeq, /* incomplete sequence */ + DR_ParamErr, /* incoming parameter error */ + DR_BufOverflow /* buffer overflow */ + /* etc. */ +} DERReturn; + +/* + * Primary representation of a block of memory. + */ +typedef struct { + DERByte *data; + DERSize length; +} DERItem; + +/* + * The structure of a sequence during decode or encode is expressed as + * an array of DERItemSpecs. While decoding or encoding a sequence, + * each item in the sequence corresponds to one DERItemSpec. + */ +typedef struct { + DERSize offset; /* offset of destination DERItem */ + DERTag tag; /* DER tag */ + DERShort options; /* DER_DEC_xxx or DER_ENC_xxx */ +} DERItemSpec; + +/* + * Macro to obtain offset of a DERDecodedInfo within a struct. + * FIXME this is going to need reworking to avoid compiler warnings + * on 64-bit compiles. It'll work OK as long as an offset can't be larger + * than a DERSize, but the cast from a pointer to a DERSize may + * provoke compiler warnings. + */ +#define DER_OFFSET(type, field) ((DERSize)(&((type *)0)->field)) + +#ifdef __cplusplus +} +#endif + +#endif /* _LIB_DER_H_ */ + diff --git a/view/kernelcache/core/transformers/libDER/libDER_config.h b/view/kernelcache/core/transformers/libDER/libDER_config.h new file mode 100644 index 00000000..08fe40da --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/libDER_config.h @@ -0,0 +1,113 @@ +/* + * Copyright (c) 2005-2007,2011-2012,2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * libDER_config.h - platform dependent #defines and typedefs for libDER + * + */ + +#ifndef _LIB_DER_CONFIG_H_ +#define _LIB_DER_CONFIG_H_ + +#include <stdint.h> +#include <string.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/* + * Basic data types: unsigned 8-bit integer, unsigned 32-bit integer + */ +typedef uint8_t DERByte; +typedef uint16_t DERShort; +typedef uint32_t DERSize; + +/* + * Use these #defines of you have memset, memmove, and memcmp; else + * write your own equivalents. + */ + +#define DERMemset(ptr, c, len) memset(ptr, c, len) +#define DERMemmove(dst, src, len) memmove(dst, src, len) +#define DERMemcmp(b1, b2, len) memcmp(b1, b2, len) + + +/*** + *** Compile time options to trim size of the library. + ***/ + +/* enable general DER encode */ +#define DER_ENCODE_ENABLE 1 + +/* enable general DER decode */ +#define DER_DECODE_ENABLE 1 + +#ifndef DER_MULTIBYTE_TAGS +/* enable multibyte tag support. */ +#define DER_MULTIBYTE_TAGS 1 +#endif + +#ifndef DER_TAG_SIZE +/* Iff DER_MULTIBYTE_TAGS is 1 this is the sizeof(DERTag) in bytes. Note that + tags are still encoded and decoded from a minimally encoded DER + represantation. This value determines how big each DERItemSpecs is, we + choose 2 since that makes DERItemSpecs 8 bytes wide. */ +#define DER_TAG_SIZE 8 +#endif + + +/* ---------------------- Do not edit below this line ---------------------- */ + +/* + * Logical representation of a tag (the encoded representation is always in + * the minimal number of bytes). The top 3 bits encode class and method + * The remaining bits encode the tag value. To obtain smaller DERItemSpecs + * sizes, choose the smallest type that fits your needs. Most standard ASN.1 + * usage only needs single byte tags, but ocasionally custom applications + * require a larger tag namespace. + */ +#if DER_MULTIBYTE_TAGS + +#if DER_TAG_SIZE == 1 +typedef uint8_t DERTag; +#elif DER_TAG_SIZE == 2 +typedef uint16_t DERTag; +#elif DER_TAG_SIZE == 4 +typedef uint32_t DERTag; +#elif DER_TAG_SIZE == 8 +typedef uint64_t DERTag; +#else +#error DER_TAG_SIZE invalid +#endif + +#else /* DER_MULTIBYTE_TAGS */ +typedef DERByte DERTag; +#endif /* !DER_MULTIBYTE_TAGS */ + +#ifdef __cplusplus +} +#endif + +#endif /* _LIB_DER_CONFIG_H_ */ diff --git a/view/kernelcache/core/transformers/libDER/oids.c b/view/kernelcache/core/transformers/libDER/oids.c new file mode 100644 index 00000000..444457c5 --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/oids.c @@ -0,0 +1,576 @@ +/* + * Copyright (c) 2005-2009,2011-2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * oids.c - OID consts + * + */ + +#include <libDER/libDER.h> +#include <libDER/oids.h> + +#define OID_ISO_CCITT_DIR_SERVICE 85 +#define OID_DS OID_ISO_CCITT_DIR_SERVICE +#define OID_ATTR_TYPE OID_DS, 4 +#define OID_EXTENSION OID_DS, 29 +#define OID_ISO_STANDARD 40 +#define OID_ISO_MEMBER 42 +#define OID_US OID_ISO_MEMBER, 134, 72 + +#define OID_ISO_IDENTIFIED_ORG 43 +#define OID_OSINET OID_ISO_IDENTIFIED_ORG, 4 +#define OID_GOSIP OID_ISO_IDENTIFIED_ORG, 5 +#define OID_DOD OID_ISO_IDENTIFIED_ORG, 6 +#define OID_OIW OID_ISO_IDENTIFIED_ORG, 14 + +/* From the PKCS Standards */ +#define OID_RSA OID_US, 134, 247, 13 +#define OID_RSA_HASH OID_RSA, 2 +#define OID_RSA_ENCRYPT OID_RSA, 3 +#define OID_PKCS OID_RSA, 1 +#define OID_PKCS_1 OID_PKCS, 1 +#define OID_PKCS_2 OID_PKCS, 2 +#define OID_PKCS_3 OID_PKCS, 3 +#define OID_PKCS_4 OID_PKCS, 4 +#define OID_PKCS_5 OID_PKCS, 5 +#define OID_PKCS_6 OID_PKCS, 6 +#define OID_PKCS_7 OID_PKCS, 7 +#define OID_PKCS_8 OID_PKCS, 8 +#define OID_PKCS_9 OID_PKCS, 9 +#define OID_PKCS_10 OID_PKCS, 10 +#define OID_PKCS_11 OID_PKCS, 11 +#define OID_PKCS_12 OID_PKCS, 12 + +/* ANSI X9.62 */ +#define OID_ANSI_X9_62 OID_US, 206, 61 +#define OID_PUBLIC_KEY_TYPE OID_ANSI_X9_62, 2 +#define OID_EC_SIG_TYPE OID_ANSI_X9_62, 4 +#define OID_ECDSA_WITH_SHA2 OID_EC_SIG_TYPE, 3 + +/* ANSI X9.42 */ +#define OID_ANSI_X9_42 OID_US, 206, 62, 2 +#define OID_ANSI_X9_42_SCHEME OID_ANSI_X9_42, 3 +#define OID_ANSI_X9_42_NAMED_SCHEME OID_ANSI_X9_42, 4 + +/* DOD IANA Security releated objects. */ +#define OID_IANA OID_DOD, 1, 5 + +/* Kerberos PKINIT */ +#define OID_KERBv5 OID_IANA, 2 +#define OID_KERBv5_PKINIT OID_KERBv5, 3 + +/* DOD IANA Mechanisms. */ +#define OID_MECHANISMS OID_IANA, 5 + +/* PKIX */ +#define OID_PKIX OID_MECHANISMS, 7 +#define OID_PE OID_PKIX, 1 +#define OID_QT OID_PKIX, 2 +#define OID_KP OID_PKIX, 3 +#define OID_OTHER_NAME OID_PKIX, 8 +#define OID_PDA OID_PKIX, 9 +#define OID_QCS OID_PKIX, 11 +#define OID_AD OID_PKIX, 48 +#define OID_AD_OCSP OID_AD, 1 +#define OID_AD_CAISSUERS OID_AD, 2 + +/* ISAKMP */ +#define OID_ISAKMP OID_MECHANISMS, 8 + +/* ETSI */ +#define OID_ETSI 0x04, 0x00 +#define OID_ETSI_QCS 0x04, 0x00, 0x8E, 0x46, 0x01 + +#define OID_OIW_SECSIG OID_OIW, 3 + +#define OID_OIW_ALGORITHM OID_OIW_SECSIG, 2 + +/* NIST defined digest algorithm arc (2, 16, 840, 1, 101, 3, 4, 2) */ +#define OID_NIST_HASHALG 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02 + +/* + * Apple-specific OID bases + */ + +/* + * apple OBJECT IDENTIFIER ::= + * { iso(1) member-body(2) US(840) 113635 } + * + * BER = 06 06 2A 86 48 86 F7 63 + */ +#define APPLE_OID OID_US, 0x86, 0xf7, 0x63 + +/* appleDataSecurity OBJECT IDENTIFIER ::= + * { apple 100 } + * { 1 2 840 113635 100 } + * + * BER = 06 07 2A 86 48 86 F7 63 64 + */ +#define APPLE_ADS_OID APPLE_OID, 0x64 + +/* + * appleTrustPolicy OBJECT IDENTIFIER ::= + * { appleDataSecurity 1 } + * { 1 2 840 113635 100 1 } + * + * BER = 06 08 2A 86 48 86 F7 63 64 01 + */ +#define APPLE_TP_OID APPLE_ADS_OID, 1 + +/* + * appleSecurityAlgorithm OBJECT IDENTIFIER ::= + * { appleDataSecurity 2 } + * { 1 2 840 113635 100 2 } + * + * BER = 06 08 2A 86 48 86 F7 63 64 02 + */ +#define APPLE_ALG_OID APPLE_ADS_OID, 2 + +/* + * appleDotMacCertificate OBJECT IDENTIFIER ::= + * { appleDataSecurity 3 } + * { 1 2 840 113635 100 3 } + */ +#define APPLE_DOTMAC_CERT_OID APPLE_ADS_OID, 3 + +/* + * Basis of Policy OIDs for .mac TP requests + * + * dotMacCertificateRequest OBJECT IDENTIFIER ::= + * { appleDotMacCertificate 1 } + * { 1 2 840 113635 100 3 1 } + */ +#define APPLE_DOTMAC_CERT_REQ_OID APPLE_DOTMAC_CERT_OID, 1 + +/* + * Basis of .mac Certificate Extensions + * + * dotMacCertificateExtension OBJECT IDENTIFIER ::= + * { appleDotMacCertificate 2 } + * { 1 2 840 113635 100 3 2 } + */ +#define APPLE_DOTMAC_CERT_EXTEN_OID APPLE_DOTMAC_CERT_OID, 2 + +/* + * Basis of .mac Certificate request OID/value identitifiers + * + * dotMacCertificateRequestValues OBJECT IDENTIFIER ::= + * { appleDotMacCertificate 3 } + * { 1 2 840 113635 100 3 3 } + */ +#define APPLE_DOTMAC_CERT_REQ_VALUE_OID APPLE_DOTMAC_CERT_OID, 3 + +/* + * Basis of Apple-specific extended key usages + * + * appleExtendedKeyUsage OBJECT IDENTIFIER ::= + * { appleDataSecurity 4 } + * { 1 2 840 113635 100 4 } + */ +#define APPLE_EKU_OID APPLE_ADS_OID, 4 + +/* + * Basis of Apple Code Signing extended key usages + * appleCodeSigning OBJECT IDENTIFIER ::= + * { appleExtendedKeyUsage 1 } + * { 1 2 840 113635 100 4 1} + */ +#define APPLE_EKU_CODE_SIGNING APPLE_EKU_OID, 1 +#define APPLE_EKU_APPLE_ID APPLE_EKU_OID, 7 +#define APPLE_EKU_SHOEBOX APPLE_EKU_OID, 14 +#define APPLE_EKU_PROFILE_SIGNING APPLE_EKU_OID, 16 +#define APPLE_EKU_QA_PROFILE_SIGNING APPLE_EKU_OID, 17 + + +/* + * Basis of Apple-specific Certificate Policy IDs. + * appleCertificatePolicies OBJECT IDENTIFIER ::= + * { appleDataSecurity 5 } + * { 1 2 840 113635 100 5 } + */ +#define APPLE_CERT_POLICIES APPLE_ADS_OID, 5 + +#define APPLE_CERT_POLICY_MOBILE_STORE APPLE_CERT_POLICIES, 12 + +#define APPLE_CERT_POLICY_TEST_MOBILE_STORE APPLE_CERT_POLICY_MOBILE_STORE, 1 + +/* + * Basis of Apple-specific Signing extensions + * { appleDataSecurity 6 } + */ +#define APPLE_CERT_EXT APPLE_ADS_OID, 6 + +/* Apple Intermediate Marker OIDs */ +#define APPLE_CERT_EXT_INTERMEDIATE_MARKER APPLE_CERT_EXT, 2 +/* Apple Apple ID Intermediate Marker */ +#define APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID APPLE_CERT_EXT_INTERMEDIATE_MARKER, 3 +/* + * Apple Apple ID Intermediate Marker (New subCA, no longer shared with push notification server cert issuer + * + * appleCertificateExtensionAppleIDIntermediate ::= + * { appleCertificateExtensionIntermediateMarker 7 } + * { 1 2 840 113635 100 6 2 7 } + */ +#define APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID_2 APPLE_CERT_EXT_INTERMEDIATE_MARKER, 7 + +#define APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID_SYSTEM_INTEGRATION_2 APPLE_CERT_EXT_INTERMEDIATE_MARKER, 10 + +#define APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID_SYSTEM_INTEGRATION_G3 APPLE_CERT_EXT_INTERMEDIATE_MARKER, 13 + +#define APPLE_CERT_EXT_APPLE_PUSH_MARKER APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID, 2 + + +#define APPLE_CERT_EXTENSION_CODESIGNING APPLE_CERT_EXT, 1 + +/* Secure Boot Embedded Image3 value, + co-opted by desktop for "Apple Released Code Signature", without value */ +#define APPLE_SBOOT_CERT_EXTEN_SBOOT_SPEC_OID APPLE_CERT_EXTENSION_CODESIGNING, 1 +/* iPhone Provisioning Profile Signing leaf - on the intermediate marker arc? */ +#define APPLE_PROVISIONING_PROFILE_OID APPLE_CERT_EXT_INTERMEDIATE_MARKER, 1 +/* iPhone Application Signing leaf */ +#define APPLE_APP_SIGNING_OID APPLE_CERT_EXTENSION_CODESIGNING, 3 + +#define APPLE_INSTALLER_PACKAGE_SIGNING_EXTERNAL_OID APPLE_CERT_EXTENSION_CODESIGNING, 16 + +#define APPLE_ESCROW_ARC APPLE_CERT_EXT, 23 + +#define APPLE_ESCROW_POLICY_OID APPLE_ESCROW_ARC, 1 + +#define APPLE_CERT_EXT_APPLE_ID_VALIDATION_RECORD_SIGNING APPLE_CERT_EXT, 25 + +#define APPLE_SERVER_AUTHENTICATION APPLE_CERT_EXT, 27 +#define APPLE_CERT_EXT_APPLE_SERVER_AUTHENTICATION APPLE_SERVER_AUTHENTICATION, 1 +#define APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLE_SERVER_AUTHENTICATION APPLE_CERT_EXT_INTERMEDIATE_MARKER, 12 + +#define APPLE_CERT_EXT_APPLE_SMP_ENCRYPTION APPLE_CERT_EXT, 30 + +/* + * Netscape OIDs. + */ +#define NETSCAPE_BASE_OID 0x60, 0x86, 0x48, 0x01, 0x86, 0xf8, 0x42 + +/* + * Netscape cert extension. + * + * netscape-cert-extension OBJECT IDENTIFIER ::= + * { 2 16 840 1 113730 1 } + * + * BER = 06 08 60 86 48 01 86 F8 42 01 + */ +#define NETSCAPE_CERT_EXTEN NETSCAPE_BASE_OID, 0x01 + +#define NETSCAPE_CERT_POLICY NETSCAPE_BASE_OID, 0x04 + +/* Entrust OIDs. */ +#define ENTRUST_BASE_OID OID_US, 0x86, 0xf6, 0x7d + +/* + * Entrust cert extension. + * + * entrust-cert-extension OBJECT IDENTIFIER ::= + * { 1 2 840 113533 7 65 } + * + * BER = 06 08 2A 86 48 86 F6 7D 07 41 + */ +#define ENTRUST_CERT_EXTEN ENTRUST_BASE_OID, 0x07, 0x41 + +/* Microsfot OIDs. */ +#define MICROSOFT_BASE_OID OID_DOD, 0x01, 0x04, 0x01, 0x82, 0x37 +#define MICROSOFT_ENROLLMENT_OID MICROSOFT_BASE_OID, 0x14 + +/* Algorithm OIDs. */ +static const DERByte + _oidRsa[] = { OID_PKCS_1, 1 }, + _oidMd2Rsa[] = { OID_PKCS_1, 2 }, + _oidMd5Rsa[] = { OID_PKCS_1, 4 }, + _oidSha1Rsa[] = { OID_PKCS_1, 5 }, + _oidSha256Rsa[] = { OID_PKCS_1, 11 }, + _oidEcPubKey[] = { OID_PUBLIC_KEY_TYPE, 1 }, + _oidSha1Ecdsa[] = { OID_EC_SIG_TYPE, 1 }, /* rfc3279 */ + _oidSha224Ecdsa[] = { OID_ECDSA_WITH_SHA2, 1 }, /* rfc5758 */ + _oidSha256Ecdsa[] = { OID_ECDSA_WITH_SHA2, 2 }, /* rfc5758 */ + _oidSha384Ecdsa[] = { OID_ECDSA_WITH_SHA2, 3 }, /* rfc5758 */ + _oidSha512Ecdsa[] = { OID_ECDSA_WITH_SHA2, 4 }, /* rfc5758 */ + _oidMd2[] = { OID_RSA_HASH, 2 }, + _oidMd4[] = { OID_RSA_HASH, 4 }, + _oidMd5[] = { OID_RSA_HASH, 5 }, + _oidSha1[] = { OID_OIW_ALGORITHM, 26 }, + _oidSha256[] = { OID_NIST_HASHALG, 1 }, + _oidSha384[] = { OID_NIST_HASHALG, 2 }, + _oidSha512[] = { OID_NIST_HASHALG, 3 }, + _oidSha224[] = { OID_NIST_HASHALG, 4 }; + +const DERItem + oidRsa = { (DERByte *)_oidRsa, + sizeof(_oidRsa) }, + oidMd2Rsa = { (DERByte *)_oidMd2Rsa, + sizeof(_oidMd2Rsa) }, + oidMd5Rsa = { (DERByte *)_oidMd5Rsa, + sizeof(_oidMd5Rsa) }, + oidSha1Rsa = { (DERByte *)_oidSha1Rsa, + sizeof(_oidSha1Rsa) }, + oidSha256Rsa = { (DERByte *)_oidSha256Rsa, + sizeof(_oidSha256Rsa) }, + oidEcPubKey = { (DERByte *)_oidEcPubKey, + sizeof(_oidEcPubKey) }, + oidSha1Ecdsa = { (DERByte *)_oidSha1Ecdsa, + sizeof(_oidSha1Ecdsa) }, + oidSha224Ecdsa = { (DERByte *)_oidSha224Ecdsa, + sizeof(_oidSha224Ecdsa) }, + oidSha256Ecdsa = { (DERByte *)_oidSha256Ecdsa, + sizeof(_oidSha256Ecdsa) }, + oidSha384Ecdsa = { (DERByte *)_oidSha384Ecdsa, + sizeof(_oidSha384Ecdsa) }, + oidSha512Ecdsa = { (DERByte *)_oidSha512Ecdsa, + sizeof(_oidSha512Ecdsa) }, + oidMd2 = { (DERByte *)_oidMd2, + sizeof(_oidMd2) }, + oidMd4 = { (DERByte *)_oidMd4, + sizeof(_oidMd4) }, + oidMd5 = { (DERByte *)_oidMd5, + sizeof(_oidMd5) }, + oidSha1 = { (DERByte *)_oidSha1, + sizeof(_oidSha1) }, + oidSha256 = { (DERByte *)_oidSha256, + sizeof(_oidSha256) }, + oidSha384 = { (DERByte *)_oidSha384, + sizeof(_oidSha384) }, + oidSha512 = { (DERByte *)_oidSha512, + sizeof(_oidSha512) }, + oidSha224 = { (DERByte *)_oidSha224, + sizeof(_oidSha224) }; + +/* Extension OIDs. */ +static const DERByte + _oidSubjectKeyIdentifier[] = { OID_EXTENSION, 14 }, + _oidKeyUsage[] = { OID_EXTENSION, 15 }, + _oidPrivateKeyUsagePeriod[] = { OID_EXTENSION, 16 }, + _oidSubjectAltName[] = { OID_EXTENSION, 17 }, + _oidIssuerAltName[] = { OID_EXTENSION, 18 }, + _oidBasicConstraints[] = { OID_EXTENSION, 19 }, + _oidCrlDistributionPoints[] = { OID_EXTENSION, 31 }, + _oidCertificatePolicies[] = { OID_EXTENSION, 32 }, + _oidAnyPolicy[] = { OID_EXTENSION, 32, 0 }, + _oidPolicyMappings[] = { OID_EXTENSION, 33 }, + _oidAuthorityKeyIdentifier[] = { OID_EXTENSION, 35 }, + _oidPolicyConstraints[] = { OID_EXTENSION, 36 }, + _oidExtendedKeyUsage[] = { OID_EXTENSION, 37 }, + _oidAnyExtendedKeyUsage[] = { OID_EXTENSION, 37, 0 }, + _oidInhibitAnyPolicy[] = { OID_EXTENSION, 54 }, + _oidAuthorityInfoAccess[] = { OID_PE, 1 }, + _oidSubjectInfoAccess[] = { OID_PE, 11 }, + _oidAdOCSP[] = { OID_AD_OCSP }, + _oidAdCAIssuer[] = { OID_AD_CAISSUERS }, + _oidNetscapeCertType[] = { NETSCAPE_CERT_EXTEN, 1 }, + _oidEntrustVersInfo[] = { ENTRUST_CERT_EXTEN, 0 }, + _oidMSNTPrincipalName[] = { MICROSOFT_ENROLLMENT_OID, 2, 3 }, + /* Policy Qualifier IDs for Internet policy qualifiers. */ + _oidQtCps[] = { OID_QT, 1 }, + _oidQtUNotice[] = { OID_QT, 2 }, + /* X.501 Name IDs. */ + _oidCommonName[] = { OID_ATTR_TYPE, 3 }, + _oidCountryName[] = { OID_ATTR_TYPE, 6 }, + _oidLocalityName[] = { OID_ATTR_TYPE, 7 }, + _oidStateOrProvinceName[] = { OID_ATTR_TYPE, 8 }, + _oidOrganizationName[] = { OID_ATTR_TYPE, 10 }, + _oidOrganizationalUnitName[] = { OID_ATTR_TYPE, 11 }, + _oidDescription[] = { OID_ATTR_TYPE, 13 }, + _oidEmailAddress[] = { OID_PKCS_9, 1 }, + _oidFriendlyName[] = { OID_PKCS_9, 20 }, + _oidLocalKeyId[] = { OID_PKCS_9, 21 }, + _oidExtendedKeyUsageServerAuth[] = { OID_KP, 1 }, + _oidExtendedKeyUsageClientAuth[] = { OID_KP, 2 }, + _oidExtendedKeyUsageCodeSigning[] = { OID_KP, 3 }, + _oidExtendedKeyUsageEmailProtection[] = { OID_KP, 4 }, + _oidExtendedKeyUsageOCSPSigning[] = { OID_KP, 9 }, + _oidExtendedKeyUsageIPSec[] = { OID_ISAKMP, 2, 2 }, + _oidExtendedKeyUsageMicrosoftSGC[] = { MICROSOFT_BASE_OID, 10, 3, 3 }, + _oidExtendedKeyUsageNetscapeSGC[] = { NETSCAPE_CERT_POLICY, 1 }, + _oidAppleSecureBootCertSpec[] = { APPLE_SBOOT_CERT_EXTEN_SBOOT_SPEC_OID }, + _oidAppleProvisioningProfile[] = {APPLE_PROVISIONING_PROFILE_OID }, + _oidAppleApplicationSigning[] = { APPLE_APP_SIGNING_OID }, + _oidAppleInstallerPackagingSigningExternal[] = { APPLE_INSTALLER_PACKAGE_SIGNING_EXTERNAL_OID }, + _oidAppleExtendedKeyUsageAppleID[] = { APPLE_EKU_APPLE_ID }, + _oidAppleExtendedKeyUsageShoebox[] = { APPLE_EKU_SHOEBOX }, + _oidAppleExtendedKeyUsageProfileSigning[] = { APPLE_EKU_PROFILE_SIGNING }, + _oidAppleExtendedKeyUsageQAProfileSigning[] = { APPLE_EKU_QA_PROFILE_SIGNING }, + _oidAppleIntmMarkerAppleID[] = { APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID }, + _oidAppleIntmMarkerAppleID2[] = {APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID_2 }, + _oidApplePushServiceClient[] = { APPLE_CERT_EXT_APPLE_PUSH_MARKER, 2 }, + _oidApplePolicyMobileStore[] = { APPLE_CERT_POLICY_MOBILE_STORE }, + _oidApplePolicyTestMobileStore[] = { APPLE_CERT_POLICY_TEST_MOBILE_STORE }, + _oidApplePolicyEscrowService[] = { APPLE_ESCROW_POLICY_OID }, + _oidAppleCertExtensionAppleIDRecordValidationSigning[] = { APPLE_CERT_EXT_APPLE_ID_VALIDATION_RECORD_SIGNING }, + _oidAppleIntmMarkerAppleSystemIntg2[] = {APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID_SYSTEM_INTEGRATION_2}, + _oidAppleIntmMarkerAppleSystemIntgG3[] = {APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLEID_SYSTEM_INTEGRATION_G3}, + _oidAppleCertExtAppleSMPEncryption[] = {APPLE_CERT_EXT_APPLE_SMP_ENCRYPTION}, + _oidAppleCertExtAppleServerAuthentication[] = {APPLE_CERT_EXT_APPLE_SERVER_AUTHENTICATION}, + _oidAppleIntmMarkerAppleServerAuthentication[] = {APPLE_CERT_EXT_INTERMEDIATE_MARKER_APPLE_SERVER_AUTHENTICATION}; + +const DERItem + oidSubjectKeyIdentifier = { (DERByte *)_oidSubjectKeyIdentifier, + sizeof(_oidSubjectKeyIdentifier) }, + oidKeyUsage = { (DERByte *)_oidKeyUsage, + sizeof(_oidKeyUsage) }, + oidPrivateKeyUsagePeriod = { (DERByte *)_oidPrivateKeyUsagePeriod, + sizeof(_oidPrivateKeyUsagePeriod) }, + oidSubjectAltName = { (DERByte *)_oidSubjectAltName, + sizeof(_oidSubjectAltName) }, + oidIssuerAltName = { (DERByte *)_oidIssuerAltName, + sizeof(_oidIssuerAltName) }, + oidBasicConstraints = { (DERByte *)_oidBasicConstraints, + sizeof(_oidBasicConstraints) }, + oidCrlDistributionPoints = { (DERByte *)_oidCrlDistributionPoints, + sizeof(_oidCrlDistributionPoints) }, + oidCertificatePolicies = { (DERByte *)_oidCertificatePolicies, + sizeof(_oidCertificatePolicies) }, + oidAnyPolicy = { (DERByte *)_oidAnyPolicy, + sizeof(_oidAnyPolicy) }, + oidPolicyMappings = { (DERByte *)_oidPolicyMappings, + sizeof(_oidPolicyMappings) }, + oidAuthorityKeyIdentifier = { (DERByte *)_oidAuthorityKeyIdentifier, + sizeof(_oidAuthorityKeyIdentifier) }, + oidPolicyConstraints = { (DERByte *)_oidPolicyConstraints, + sizeof(_oidPolicyConstraints) }, + oidExtendedKeyUsage = { (DERByte *)_oidExtendedKeyUsage, + sizeof(_oidExtendedKeyUsage) }, + oidAnyExtendedKeyUsage = { (DERByte *)_oidAnyExtendedKeyUsage, + sizeof(_oidAnyExtendedKeyUsage) }, + oidInhibitAnyPolicy = { (DERByte *)_oidInhibitAnyPolicy, + sizeof(_oidInhibitAnyPolicy) }, + oidAuthorityInfoAccess = { (DERByte *)_oidAuthorityInfoAccess, + sizeof(_oidAuthorityInfoAccess) }, + oidSubjectInfoAccess = { (DERByte *)_oidSubjectInfoAccess, + sizeof(_oidSubjectInfoAccess) }, + oidAdOCSP = { (DERByte *)_oidAdOCSP, + sizeof(_oidAdOCSP) }, + oidAdCAIssuer = { (DERByte *)_oidAdCAIssuer, + sizeof(_oidAdCAIssuer) }, + oidNetscapeCertType = { (DERByte *)_oidNetscapeCertType, + sizeof(_oidNetscapeCertType) }, + oidEntrustVersInfo = { (DERByte *)_oidEntrustVersInfo, + sizeof(_oidEntrustVersInfo) }, + oidMSNTPrincipalName = { (DERByte *)_oidMSNTPrincipalName, + sizeof(_oidMSNTPrincipalName) }, + /* Policy Qualifier IDs for Internet policy qualifiers. */ + oidQtCps = { (DERByte *)_oidQtCps, + sizeof(_oidQtCps) }, + oidQtUNotice = { (DERByte *)_oidQtUNotice, + sizeof(_oidQtUNotice) }, + /* X.501 Name IDs. */ + oidCommonName = { (DERByte *)_oidCommonName, + sizeof(_oidCommonName) }, + oidCountryName = { (DERByte *)_oidCountryName, + sizeof(_oidCountryName) }, + oidLocalityName = { (DERByte *)_oidLocalityName, + sizeof(_oidLocalityName) }, + oidStateOrProvinceName = { (DERByte *)_oidStateOrProvinceName, + sizeof(_oidStateOrProvinceName) }, + oidOrganizationName = { (DERByte *)_oidOrganizationName, + sizeof(_oidOrganizationName) }, + oidOrganizationalUnitName = { (DERByte *)_oidOrganizationalUnitName, + sizeof(_oidOrganizationalUnitName) }, + oidDescription = { (DERByte *)_oidDescription, + sizeof(_oidDescription) }, + oidEmailAddress = { (DERByte *)_oidEmailAddress, + sizeof(_oidEmailAddress) }, + oidFriendlyName = { (DERByte *)_oidFriendlyName, + sizeof(_oidFriendlyName) }, + oidLocalKeyId = { (DERByte *)_oidLocalKeyId, + sizeof(_oidLocalKeyId) }, + oidExtendedKeyUsageServerAuth = { (DERByte *)_oidExtendedKeyUsageServerAuth, + sizeof(_oidExtendedKeyUsageServerAuth) }, + oidExtendedKeyUsageClientAuth = { (DERByte *)_oidExtendedKeyUsageClientAuth, + sizeof(_oidExtendedKeyUsageClientAuth) }, + oidExtendedKeyUsageCodeSigning = { (DERByte *)_oidExtendedKeyUsageCodeSigning, + sizeof(_oidExtendedKeyUsageCodeSigning) }, + oidExtendedKeyUsageEmailProtection = { (DERByte *)_oidExtendedKeyUsageEmailProtection, + sizeof(_oidExtendedKeyUsageEmailProtection) }, + oidExtendedKeyUsageOCSPSigning = { (DERByte *)_oidExtendedKeyUsageOCSPSigning, + sizeof(_oidExtendedKeyUsageOCSPSigning) }, + oidExtendedKeyUsageIPSec = { (DERByte *)_oidExtendedKeyUsageIPSec, + sizeof(_oidExtendedKeyUsageIPSec) }, + oidExtendedKeyUsageMicrosoftSGC = { (DERByte *)_oidExtendedKeyUsageMicrosoftSGC, + sizeof(_oidExtendedKeyUsageMicrosoftSGC) }, + oidExtendedKeyUsageNetscapeSGC = { (DERByte *)_oidExtendedKeyUsageNetscapeSGC, + sizeof(_oidExtendedKeyUsageNetscapeSGC) }, + oidAppleSecureBootCertSpec = { (DERByte *)_oidAppleSecureBootCertSpec, + sizeof(_oidAppleSecureBootCertSpec) }, + oidAppleProvisioningProfile = { (DERByte *)_oidAppleProvisioningProfile, + sizeof(_oidAppleProvisioningProfile) }, + oidAppleApplicationSigning = { (DERByte *)_oidAppleApplicationSigning, + sizeof(_oidAppleApplicationSigning) }, + oidAppleInstallerPackagingSigningExternal = { (DERByte *)_oidAppleInstallerPackagingSigningExternal, + sizeof(_oidAppleInstallerPackagingSigningExternal) }, + oidAppleExtendedKeyUsageAppleID = { (DERByte *)_oidAppleExtendedKeyUsageAppleID, + sizeof(_oidAppleExtendedKeyUsageAppleID) }, + oidAppleExtendedKeyUsageShoebox = { (DERByte *)_oidAppleExtendedKeyUsageShoebox, + sizeof(_oidAppleExtendedKeyUsageShoebox) }, + oidAppleExtendedKeyUsageProfileSigning + = { (DERByte *)_oidAppleExtendedKeyUsageProfileSigning, + sizeof(_oidAppleExtendedKeyUsageProfileSigning) }, + oidAppleExtendedKeyUsageQAProfileSigning + = { (DERByte *)_oidAppleExtendedKeyUsageQAProfileSigning, + sizeof(_oidAppleExtendedKeyUsageQAProfileSigning) }, + oidAppleIntmMarkerAppleID = { (DERByte *)_oidAppleIntmMarkerAppleID, + sizeof(_oidAppleIntmMarkerAppleID) }, + oidAppleIntmMarkerAppleID2 = { (DERByte *)_oidAppleIntmMarkerAppleID2, + sizeof(_oidAppleIntmMarkerAppleID2) }, + oidApplePushServiceClient = { (DERByte *)_oidAppleIntmMarkerAppleID2, + sizeof(_oidAppleIntmMarkerAppleID2) }, + oidApplePolicyMobileStore = { (DERByte *)_oidApplePolicyMobileStore, + sizeof(_oidApplePolicyMobileStore)}, + oidApplePolicyTestMobileStore = { (DERByte *)_oidApplePolicyTestMobileStore, + sizeof(_oidApplePolicyTestMobileStore)}, + oidApplePolicyEscrowService = { (DERByte *)_oidApplePolicyEscrowService, + sizeof(_oidApplePolicyEscrowService)}, + oidAppleCertExtensionAppleIDRecordValidationSigning = { (DERByte *)_oidAppleCertExtensionAppleIDRecordValidationSigning, + sizeof(_oidAppleCertExtensionAppleIDRecordValidationSigning)}, + oidAppleIntmMarkerAppleSystemIntg2 = { (DERByte *) _oidAppleIntmMarkerAppleSystemIntg2, + sizeof(_oidAppleIntmMarkerAppleSystemIntg2)}, + oidAppleIntmMarkerAppleSystemIntgG3 = { (DERByte *) _oidAppleIntmMarkerAppleSystemIntgG3, + sizeof(_oidAppleIntmMarkerAppleSystemIntgG3)}, + oidAppleCertExtAppleSMPEncryption = { (DERByte *)_oidAppleCertExtAppleSMPEncryption, + sizeof(_oidAppleCertExtAppleSMPEncryption)}, + oidAppleCertExtAppleServerAuthentication + = { (DERByte *)_oidAppleCertExtAppleServerAuthentication, + sizeof(_oidAppleCertExtAppleServerAuthentication) }, + oidAppleIntmMarkerAppleServerAuthentication + = { (DERByte *)_oidAppleIntmMarkerAppleServerAuthentication, + sizeof(_oidAppleIntmMarkerAppleServerAuthentication) }; + + +bool DEROidCompare(const DERItem *oid1, const DERItem *oid2) { + if ((oid1 == NULL) || (oid2 == NULL)) { + return false; + } + if (oid1->length != oid2->length) { + return false; + } + if (!DERMemcmp(oid1->data, oid2->data, oid1->length)) { + return true; + } else { + return false; + } +} diff --git a/view/kernelcache/core/transformers/libDER/oids.h b/view/kernelcache/core/transformers/libDER/oids.h new file mode 100644 index 00000000..33ddabca --- /dev/null +++ b/view/kernelcache/core/transformers/libDER/oids.h @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2005-2009,2011-2014 Apple Inc. All Rights Reserved. + * + * @APPLE_LICENSE_HEADER_START@ + * + * This file contains Original Code and/or Modifications of Original Code + * as defined in and that are subject to the Apple Public Source License + * Version 2.0 (the 'License'). You may not use this file except in + * compliance with the License. Please obtain a copy of the License at + * http://www.opensource.apple.com/apsl/ and read it before using this + * file. + * + * The Original Code and all software distributed under the License are + * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER + * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, + * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. + * Please see the License for the specific language governing rights and + * limitations under the License. + * + * @APPLE_LICENSE_HEADER_END@ + */ + + +/* + * oids.h - declaration of OID consts + * + */ + +#ifndef _LIB_DER_OIDS_H_ +#define _LIB_DER_OIDS_H_ + +#include <libDER/libDER.h> +#include <stdbool.h> + +#ifdef __cplusplus +extern "C" { +#endif + +/* Algorithm oids. */ +extern const DERItem + oidRsa, /* PKCS1 RSA encryption, used to identify RSA keys */ + oidMd2Rsa, /* PKCS1 md2withRSAEncryption signature alg */ + oidMd5Rsa, /* PKCS1 md5withRSAEncryption signature alg */ + oidSha1Rsa, /* PKCS1 sha1withRSAEncryption signature alg */ + oidSha256Rsa, /* PKCS1 sha256WithRSAEncryption signature alg */ + oidEcPubKey, /* ECDH or ECDSA public key in a certificate */ + oidSha1Ecdsa, /* ECDSA with SHA1 signature alg */ + oidSha224Ecdsa, /* ECDSA with SHA224 signature alg */ + oidSha256Ecdsa, /* ECDSA with SHA256 signature alg */ + oidSha384Ecdsa, /* ECDSA with SHA384 signature alg */ + oidSha512Ecdsa, /* ECDSA with SHA512 signature alg */ + oidMd2, /* OID_RSA_HASH 2 */ + oidMd4, /* OID_RSA_HASH 4 */ + oidMd5, /* OID_RSA_HASH 5 */ + oidSha1, /* OID_OIW_ALGORITHM 26 */ + oidSha256, /* OID_NIST_HASHALG 1 */ + oidSha384, /* OID_NIST_HASHALG 2 */ + oidSha512, /* OID_NIST_HASHALG 3 */ + oidSha224; /* OID_NIST_HASHALG 4 */ + +/* Standard X.509 Cert and CRL extensions. */ +extern const DERItem + oidSubjectKeyIdentifier, + oidKeyUsage, + oidPrivateKeyUsagePeriod, + oidSubjectAltName, + oidIssuerAltName, + oidBasicConstraints, + oidCrlDistributionPoints, + oidCertificatePolicies, + oidAnyPolicy, + oidPolicyMappings, + oidAuthorityKeyIdentifier, + oidPolicyConstraints, + oidExtendedKeyUsage, + oidAnyExtendedKeyUsage, + oidInhibitAnyPolicy, + oidAuthorityInfoAccess, + oidSubjectInfoAccess, + oidAdOCSP, + oidAdCAIssuer, + oidNetscapeCertType, + oidEntrustVersInfo, + oidMSNTPrincipalName, + /* Policy Qualifier IDs for Internet policy qualifiers. */ + oidQtCps, + oidQtUNotice, + /* X.501 Name IDs. */ + oidCommonName, + oidCountryName, + oidLocalityName, + oidStateOrProvinceName, + oidOrganizationName, + oidOrganizationalUnitName, + oidDescription, + oidEmailAddress, + oidFriendlyName, + oidLocalKeyId, + oidExtendedKeyUsageServerAuth, + oidExtendedKeyUsageClientAuth, + oidExtendedKeyUsageCodeSigning, + oidExtendedKeyUsageEmailProtection, + oidExtendedKeyUsageOCSPSigning, + oidExtendedKeyUsageIPSec, + oidExtendedKeyUsageMicrosoftSGC, + oidExtendedKeyUsageNetscapeSGC, + /* Secure Boot Spec oid */ + oidAppleSecureBootCertSpec, + oidAppleProvisioningProfile, + oidAppleApplicationSigning, + oidAppleInstallerPackagingSigningExternal, + oidAppleExtendedKeyUsageAppleID, + oidAppleExtendedKeyUsageShoebox, + oidAppleExtendedKeyUsageProfileSigning, + oidAppleExtendedKeyUsageQAProfileSigning, + oidAppleIntmMarkerAppleID, + oidAppleIntmMarkerAppleID2, + oidApplePushServiceClient, + oidApplePolicyMobileStore, + oidApplePolicyTestMobileStore, + oidApplePolicyEscrowService, + oidAppleCertExtensionAppleIDRecordValidationSigning, + oidAppleIntmMarkerAppleSystemIntg2, + oidAppleIntmMarkerAppleSystemIntgG3, + oidAppleCertExtAppleSMPEncryption, + oidAppleCertExtAppleServerAuthentication, + oidAppleIntmMarkerAppleServerAuthentication; + +/* Compare two decoded OIDs. Returns true iff they are equivalent. */ +bool DEROidCompare(const DERItem *oid1, const DERItem *oid2); + +#ifdef __cplusplus +} +#endif + +#endif /* _LIB_DER_UTILS_H_ */ diff --git a/view/kernelcache/core/transformers/libimg4/img4.c b/view/kernelcache/core/transformers/libimg4/img4.c new file mode 100644 index 00000000..9ad91049 --- /dev/null +++ b/view/kernelcache/core/transformers/libimg4/img4.c @@ -0,0 +1,383 @@ +/* + * pongoOS - https://checkra.in + * + * Copyright (C) 2021 checkra1n team + * + * This file is part of pongoOS. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ +#include <stdlib.h> +#include <stddef.h> +#include "img4.h" + +// ==================== ==================== ==================== Data ==================== ==================== ==================== + +const DERItemSpec DERImg4ItemSpecs[] = + { + {0 * sizeof(DERItem), ASN1_IA5_STRING, 0}, // IMG4 + {1 * sizeof(DERItem), ASN1_CONSTR_SEQUENCE, DER_DEC_SAVE_DER | DER_ENC_WRITE_DER}, // Payload + {2 * sizeof(DERItem), ASN1_CONSTR_CONT | 0, DER_DEC_OPTIONAL}, // Manifest + {3 * sizeof(DERItem), ASN1_CONSTR_CONT | 1, DER_DEC_OPTIONAL}, // RestoreInfo + }; +const DERItemSpec DERImg4PayloadItemSpecs[] = + { + {offsetof(Img4Payload, magic), ASN1_IA5_STRING, 0}, // IM4P + {offsetof(Img4Payload, type), ASN1_IA5_STRING, 0}, // Type + {offsetof(Img4Payload, version), ASN1_IA5_STRING, 0}, // Version + {offsetof(Img4Payload, payload), ASN1_OCTET_STRING, 0}, // Payload + {offsetof(Img4Payload, keybag), ASN1_OCTET_STRING, DER_DEC_OPTIONAL}, // Keybag + {offsetof(Img4Payload, compression), ASN1_CONSTR_SEQUENCE, DER_DEC_OPTIONAL}, // Compression + }; +const DERItemSpec DERImg4ManifestItemSpecs[] = + { + {offsetof(Img4Manifest, magic), ASN1_IA5_STRING, 0}, // IM4M + {offsetof(Img4Manifest, zero), ASN1_INTEGER, 0}, // 0 + {offsetof(Img4Manifest, properties), ASN1_CONSTR_SET, + DER_DEC_SAVE_DER | DER_ENC_WRITE_DER}, // Properties + {offsetof(Img4Manifest, signature), ASN1_OCTET_STRING, 0}, // Signature + {offsetof(Img4Manifest, certificates), ASN1_CONSTR_SEQUENCE, 0}, // Certificates + }; +const DERItemSpec DERImg4RestoreInfoItemSpecs[] = + { + {offsetof(Img4RestoreInfo, magic), ASN1_IA5_STRING, 0}, // IM4R + {offsetof(Img4RestoreInfo, nonce), ASN1_CONSTR_SET, 0}, // Nonce + }; +const DERItemSpec DERImg4NonceItemSpecs[] = + { + {0 * sizeof(DERItem), ASN1_IA5_STRING, 0}, // BNCN + {1 * sizeof(DERItem), ASN1_OCTET_STRING, 0}, // Nonce + }; +const DERItemSpec DERImg4CompressionItemSpecs[] = + { + {0 * sizeof(DERItem), ASN1_INTEGER, 0}, + {1 * sizeof(DERItem), ASN1_INTEGER, 0}, + }; + +// ==================== ==================== ==================== Code ==================== ==================== ==================== + +DERReturn DERImg4DecodeFindInSequence(DERByte *nextItem, DERByte *end, DERTag tag, DERItem *out) { + DERDecodedInfo decoded; + DERSequence seq = {nextItem, end}; + do { + DERReturn ret = DERDecodeSeqNext(&seq, &decoded); + if (ret != DR_Success) { + return ret; + } + } while (decoded.tag != tag); + *out = decoded.content; + return DR_Success; +} + +DERReturn DERImg4DecodeContentFindItemWithTag(const DERItem *der, DERTag tag, DERItem *out) { + DERSequence seq; + DERReturn ret = DERDecodeSeqContentInit(der, &seq); + if (ret != DR_Success) { + return ret; + } + return DERImg4DecodeFindInSequence(seq.nextItem, seq.end, tag, out); +} + +DERReturn DERImg4DecodeTagCompare(const DERItem *der, uint32_t name) { + if (der->length < 4) { + return -1; + } + if (der->length > 4) { + return 1; + } + uint32_t value; + if (DERParseInteger(der, &value) != DR_Success) { + return -2; + } + if (value < name) { + return -1; + } + if (value > name) { + return 1; + } + return 0; +} + +DERReturn DERImg4Decode(const DERItem *der, DERItem *items) { + DERReturn ret; + DERDecodedInfo decoded; + if (!der || !items) { + return DR_ParamErr; + } + ret = DERDecodeItem(der, &decoded); + if (ret != DR_Success) { + return ret; + } + if (decoded.tag != ASN1_CONSTR_SEQUENCE) { + return DR_UnexpectedTag; + } + if (der->data + der->length != decoded.content.data + decoded.content.length) { + return DR_BufOverflow; + } + ret = DERParseSequenceContent(&decoded.content, sizeof(DERImg4ItemSpecs) / sizeof(DERImg4ItemSpecs[0]), + DERImg4ItemSpecs, items, 0); + if (ret != DR_Success) { + return ret; + } + if (DERImg4DecodeTagCompare(&items[0], 'IMG4') != 0) { + return DR_UnexpectedTag; + } + return DR_Success; +} +DERReturn DERImg4DecodePayload(const DERItem *der, Img4Payload *payload) { + if (!der || !payload) { + return DR_ParamErr; + } + DERReturn ret = DERParseSequence(der, sizeof(DERImg4PayloadItemSpecs) / sizeof(DERImg4PayloadItemSpecs[0]), + DERImg4PayloadItemSpecs, payload, 0); + if (ret != DR_Success) { + return ret; + } + if (DERImg4DecodeTagCompare(&payload->magic, 'IM4P') != 0) { + return DR_UnexpectedTag; + } + if (!payload->compression.data) { + return DR_Success; + } + uint32_t val = -1; + + DERItem items[2]; + ret = DERParseSequenceContent(&payload->compression, + sizeof(DERImg4CompressionItemSpecs) / sizeof(DERImg4CompressionItemSpecs[0]), + DERImg4CompressionItemSpecs, items, 0); + if (ret != DR_Success) { + return ret; + } + ret = DERParseInteger(&items[0], &val); + if (ret != DR_Success) { + return ret; + } + if (val >= 2) { + return DR_ParamErr; + } + return DR_Success; +} + +DERReturn DERImg4DecodeManifest(const DERItem *der, Img4Manifest *manifest) { + if (!der || !manifest) { + return DR_ParamErr; + } + if (!der->data || !der->length) { + return DR_Success; + } + DERReturn ret = DERParseSequence(der, sizeof(DERImg4ManifestItemSpecs) / sizeof(DERImg4ManifestItemSpecs[0]), + DERImg4ManifestItemSpecs, manifest, 0); + if (ret != DR_Success) { + return ret; + } + if (DERImg4DecodeTagCompare(&manifest->magic, 'IM4M') != 0) { + return DR_UnexpectedTag; + } + uint32_t zero; + ret = DERParseInteger(&manifest->zero, &zero); + if (ret != DR_Success) { + return ret; + } + if (zero != 0) { + return DR_UnexpectedTag; + } + return DR_Success; +} + +DERReturn DERImg4DecodeRestoreInfo(const DERItem *der, Img4RestoreInfo *restoreInfo) { + if (!der) { + return DR_Success; + } + if (!restoreInfo) { + return DR_ParamErr; + } + if (!der->data || !der->length) { + return DR_Success; + } + DERReturn ret = DERParseSequence(der, sizeof(DERImg4RestoreInfoItemSpecs) / sizeof(DERImg4RestoreInfoItemSpecs[0]), + DERImg4RestoreInfoItemSpecs, restoreInfo, 0); + if (ret != DR_Success) { + return ret; + } + if (DERImg4DecodeTagCompare(&restoreInfo->magic, 'IM4R') != 0) { + return DR_UnexpectedTag; + } + return DR_Success; +} + +DERReturn DERImg4DecodeFindProperty(const DERItem *der, DERTag ktag, DERTag vtag, Img4Property *prop) { + DERItem key; + DERReturn ret = DERImg4DecodeContentFindItemWithTag(der, ktag, &key); + if (ret != DR_Success) { + return ret; + } + DERItemSpec spec[] = + { + {0 * sizeof(Img4Property), ASN1_IA5_STRING, 0}, + {1 * sizeof(Img4Property), vtag, 0}, + }; + ret = DERParseSequence(&key, sizeof(spec) / sizeof(DERItemSpec), spec, prop, 0); + if (ret != DR_Success) { + return ret; + } + uint32_t tag; + ret = DERParseInteger(&prop[0].content, &tag); + if (ret != DR_Success) { + return ret; + } + if ((ASN1_CONSTR_PRIVATE | tag) != ktag) { + return DR_UnexpectedTag; + } + prop[0].tag = ASN1_CONSTR_PRIVATE | ktag; + prop[1].tag = vtag; + return DR_Success; +} + +DERReturn Img4DecodeGetPayload(const Img4 *img4, DERItem *item) { + if (!img4 || !item) { + return DR_ParamErr; + } + if (!img4->payload.payload.data || !img4->payload.payload.length) { + return DR_EndOfSequence; + } + *item = img4->payload.payload; + return DR_Success; +} + +DERReturn Img4DecodeGetPayloadType(const Img4 *img4, uint32_t *type) { + if (!img4 || !type) { + return DR_ParamErr; + } + if (!img4->payload.payload.data || !img4->payload.payload.length) { + return DR_EndOfSequence; + } + return DERParseInteger(&img4->payload.type, type); +} + +DERReturn Img4DecodeGetPayloadKeybag(const Img4 *img4, DERItem *kbag) { + if (!img4 || !kbag) { + return DR_ParamErr; + } + if (!img4->payload.payload.data || !img4->payload.payload.length) { + return DR_EndOfSequence; + } + *kbag = img4->payload.keybag; + return DR_Success; +} + +DERReturn Img4DecodeInit(const DERByte *data, DERSize length, Img4 *img4) { + DERReturn ret; + if (!data || !img4) { + return DR_ParamErr; + } + DERItem der = {.data = (DERByte *) data, .length = length}; + DERItem items[4] = {0}; + memset(img4, 0, sizeof(Img4)); + ret = DERImg4Decode(&der, items); + if (ret != DR_Success) { + return ret; + } + ret = DERImg4DecodePayload(&items[1], &img4->payload); + if (ret != DR_Success) { + return ret; + } + ret = DERImg4DecodeManifest(&items[2], &img4->manifest); + if (ret != DR_Success) { + return ret; + } + ret = DERImg4DecodeRestoreInfo(&items[3], &img4->restoreInfo); + if (ret != DR_Success) { + return ret; + } + img4->payloadRaw = items[1]; + img4->manifestRaw = items[2]; + return DR_Success; +} + +DERReturn Img4Encode(DERItem *der, const DERItem *items) { + return Img4EncodeSequence(ASN1_CONSTR_SEQUENCE, items, sizeof(DERImg4ItemSpecs) / sizeof(DERImg4ItemSpecs[0]), + DERImg4ItemSpecs, der); +} + +DERReturn Img4EncodeRestoreInfo(DERItem *der, void *bytes, size_t len) { + Img4RestoreInfo restoreInfo = + { + .magic = {.data = (DERByte *) "IM4R", .length = 4}, + }; + DERItem bncn; + DERItem items[] = + { + {.data = (DERByte *) "BNCN", .length = 4}, + {.data = bytes, .length = len}, + }; + DERReturn ret = Img4EncodeSequence(ASN1_CONSTR_SEQUENCE, items, + sizeof(DERImg4NonceItemSpecs) / sizeof(DERImg4NonceItemSpecs[0]), + DERImg4NonceItemSpecs, &bncn); + if (ret == DR_Success) { + DERSize inOutLen = 20 + bncn.length; + DERByte *buf = malloc(inOutLen); + if (!buf) { + ret = -1; + } else { + ret = DEREncodeItem(ASN1_CONSTR_PRIVATE | 'BNCN', bncn.length, bncn.data, buf, &inOutLen); + if (ret == DR_Success) { + restoreInfo.nonce.data = buf; + restoreInfo.nonce.length = inOutLen; + ret = Img4EncodeSequence(ASN1_CONSTR_SEQUENCE, &restoreInfo, + sizeof(DERImg4RestoreInfoItemSpecs) / sizeof(DERImg4RestoreInfoItemSpecs[0]), + DERImg4RestoreInfoItemSpecs, der); + } + free(buf); + } + free(bncn.data); + } + return ret; +} + +DERReturn +Img4EncodeSequence(DERTag tag, const void *src, DERShort numItems, const DERItemSpec *itemSpecs, DERItem *der) { + if (!tag || !src || !numItems || !itemSpecs || !der) { + return DR_ParamErr; + } + DERSize inOutLen = 20; + for (DERShort i = 0; i < numItems; ++i) { + const DERItemSpec *spec = &itemSpecs[i]; + const DERItem *item = (const DERItem *) ((uintptr_t) src + spec->offset); + if (spec->options & DER_ENC_WRITE_DER) { + inOutLen += item->length; + } else if (item->length == 0 && (spec->options & DER_DEC_OPTIONAL)) { + // Skip + } else { + inOutLen += item->length + 20; /* 1+9 tag, 1+8 size, 1 null pad */ + } + } + DERByte *buf = malloc(inOutLen); + if (!buf) { + return -1; + } + DERReturn ret = DEREncodeSequence(tag, src, numItems, itemSpecs, buf, &inOutLen); + if (ret == DR_Success) { + der->data = buf; + der->length = inOutLen; + } else { + free(buf); + } + return ret; +}
\ No newline at end of file diff --git a/view/kernelcache/core/transformers/libimg4/img4.h b/view/kernelcache/core/transformers/libimg4/img4.h new file mode 100644 index 00000000..81bcbb02 --- /dev/null +++ b/view/kernelcache/core/transformers/libimg4/img4.h @@ -0,0 +1,132 @@ +/* + * pongoOS - https://checkra.in + * + * Copyright (C) 2021 checkra1n team + * + * This file is part of pongoOS. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * + */ +#include <stdbool.h> +#include <stdint.h> +#include "libDER/libDER_config.h" +#include "libDER/asn1Types.h" // This include MUST come after libDER_config.h +#include "libDER/libDER.h" +#include "libDER/DER_Decode.h" +#include "libDER/DER_Encode.h" + +#if !DER_MULTIBYTE_TAGS +# error "DER_MULTIBYTE_TAGS not set" +#endif +#if DER_TAG_SIZE != 8 +# error "DER_TAG_SIZE != 8" +#endif + +// ==================== ==================== ==================== Constants ==================== ==================== ==================== + +#define ASN1_CONSTR_CONT (ASN1_CONSTRUCTED | ASN1_CONTEXT_SPECIFIC) +#define ASN1_CONSTR_PRIVATE (ASN1_CONSTRUCTED | ASN1_PRIVATE) + +// ==================== ==================== ==================== Types ==================== ==================== ==================== + +#ifdef __cplusplus +extern "C" { +#endif +typedef struct { + DERItem content; + DERTag tag; +} Img4Property; + +typedef struct { + DERItem magic; + DERItem type; + DERItem version; + DERItem payload; + DERItem keybag; + DERItem compression; + uint8_t hash[0x30]; +} Img4Payload; + +typedef struct { + DERItem magic; + DERItem zero; + DERItem properties; + DERItem signature; + DERItem certificates; + DERItem embedded; + uint8_t full_hash[0x30]; + uint8_t prop_hash[0x30]; +} Img4Manifest; + +typedef struct { + DERItem magic; + DERItem nonce; +} Img4RestoreInfo; + +typedef struct { + bool payloadHashValid; + bool manifestHashValid; + DERItem payloadRaw; + DERItem manifestRaw; + DERItem manb; + DERItem manp; + DERItem objp; + Img4Payload payload; + Img4Manifest manifest; + Img4RestoreInfo restoreInfo; +} Img4; + +// ==================== ==================== ==================== Functions ==================== ==================== ==================== + +DERReturn DERImg4DecodeFindInSequence(DERByte *nextItem, DERByte *end, DERTag tag, DERItem *out); + +DERReturn DERImg4DecodeContentFindItemWithTag(const DERItem *der, DERTag tag, DERItem *out); + +DERReturn DERImg4DecodeTagCompare(const DERItem *der, uint32_t name); + +DERReturn DERImg4Decode(const DERItem *der, DERItem *items); + +DERReturn DERImg4DecodePayload(const DERItem *der, Img4Payload *payload); + +DERReturn DERImg4DecodeManifest(const DERItem *der, Img4Manifest *manifest); + +DERReturn DERImg4DecodeRestoreInfo(const DERItem *der, Img4RestoreInfo *restoreInfo); + +DERReturn DERImg4DecodeFindProperty(const DERItem *der, DERTag ktag, DERTag vtag, Img4Property *prop); + +DERReturn Img4DecodeGetPayload(const Img4 *img4, DERItem *item); + +DERReturn Img4DecodeGetPayloadType(const Img4 *img4, uint32_t *type); + +DERReturn Img4DecodeGetPayloadKeybag(const Img4 *img4, DERItem *kbag); + +DERReturn Img4DecodeInit(const DERByte *data, DERSize length, Img4 *img4); + +DERReturn Img4Encode(DERItem *der, const DERItem *items); + +DERReturn Img4EncodeRestoreInfo(DERItem *der, void *bytes, size_t len); + +DERReturn +Img4EncodeSequence(DERTag tag, const void *src, DERShort numItems, const DERItemSpec *itemSpecs, DERItem *der); + + +#ifdef __cplusplus +} +#endif
\ No newline at end of file diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse.h b/view/kernelcache/core/transformers/liblzfse/lzfse.h new file mode 100644 index 00000000..e522cbe6 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse.h @@ -0,0 +1,136 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZFSE_H +#define LZFSE_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +# define __attribute__(X) +# pragma warning(disable : 4068) +#endif + +#if defined(LZFSE_DLL) +# if defined(_WIN32) || defined(__CYGWIN__) +# if defined(LZFSE_DLL_EXPORTS) +# define LZFSE_API __declspec(dllexport) +# else +# define LZFSE_API __declspec(dllimport) +# endif +# endif +#endif + +#if !defined(LZFSE_API) +# if __GNUC__ >= 4 +# define LZFSE_API __attribute__((visibility("default"))) +# else +# define LZFSE_API +# endif +#endif + +/*! @abstract Get the required scratch buffer size to compress using LZFSE. */ +LZFSE_API size_t lzfse_encode_scratch_size(); + +/*! @abstract Compress a buffer using LZFSE. + * + * @param dst_buffer + * Pointer to the first byte of the destination buffer. + * + * @param dst_size + * Size of the destination buffer in bytes. + * + * @param src_buffer + * Pointer to the first byte of the source buffer. + * + * @param src_size + * Size of the source buffer in bytes. + * + * @param scratch_buffer + * If non-NULL, a pointer to scratch space for the routine to use as workspace; + * the routine may use up to lzfse_encode_scratch_size( ) bytes of workspace + * during its operation, and will not perform any internal allocations. If + * NULL, the routine may allocate its own memory to use during operation via + * a single call to malloc( ), and will release it by calling free( ) prior + * to returning. For most use, passing NULL is perfectly satisfactory, but if + * you require strict control over allocation, you will want to pass an + * explicit scratch buffer. + * + * @return + * The number of bytes written to the destination buffer if the input is + * successfully compressed. If the input cannot be compressed to fit into + * the provided buffer, or an error occurs, zero is returned, and the + * contents of dst_buffer are unspecified. */ +LZFSE_API size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer, + size_t dst_size, + const uint8_t *__restrict src_buffer, + size_t src_size, + void *__restrict scratch_buffer); + +/*! @abstract Get the required scratch buffer size to decompress using LZFSE. */ +LZFSE_API size_t lzfse_decode_scratch_size(); + +/*! @abstract Decompress a buffer using LZFSE. + * + * @param dst_buffer + * Pointer to the first byte of the destination buffer. + * + * @param dst_size + * Size of the destination buffer in bytes. + * + * @param src_buffer + * Pointer to the first byte of the source buffer. + * + * @param src_size + * Size of the source buffer in bytes. + * + * @param scratch_buffer + * If non-NULL, a pointer to scratch space for the routine to use as workspace; + * the routine may use up to lzfse_decode_scratch_size( ) bytes of workspace + * during its operation, and will not perform any internal allocations. If + * NULL, the routine may allocate its own memory to use during operation via + * a single call to malloc( ), and will release it by calling free( ) prior + * to returning. For most use, passing NULL is perfectly satisfactory, but if + * you require strict control over allocation, you will want to pass an + * explicit scratch buffer. + * + * @return + * The number of bytes written to the destination buffer if the input is + * successfully decompressed. If there is not enough space in the destination + * buffer to hold the entire expanded output, only the first dst_size bytes + * will be written to the buffer and dst_size is returned. Note that this + * behavior differs from that of lzfse_encode_buffer. */ +LZFSE_API size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, + size_t dst_size, + const uint8_t *__restrict src_buffer, + size_t src_size, + void *__restrict scratch_buffer); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* LZFSE_H */ diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_decode.c b/view/kernelcache/core/transformers/liblzfse/lzfse_decode.c new file mode 100644 index 00000000..8fb850d6 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_decode.c @@ -0,0 +1,72 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZFSE decode API + +#include "lzfse.h" +#include "lzfse_internal.h" + +size_t lzfse_decode_scratch_size() { return sizeof(lzfse_decoder_state); } + +size_t lzfse_decode_buffer_with_scratch(uint8_t *__restrict dst_buffer, + size_t dst_size, const uint8_t *__restrict src_buffer, + size_t src_size, void *__restrict scratch_buffer) { + lzfse_decoder_state *s = (lzfse_decoder_state *)scratch_buffer; + memset(s, 0x00, sizeof(*s)); + + // Initialize state + s->src = src_buffer; + s->src_begin = src_buffer; + s->src_end = s->src + src_size; + s->dst = dst_buffer; + s->dst_begin = dst_buffer; + s->dst_end = dst_buffer + dst_size; + + // Decode + int status = lzfse_decode(s); + if (status == LZFSE_STATUS_DST_FULL) + return dst_size; + if (status != LZFSE_STATUS_OK) + return 0; // failed + return (size_t)(s->dst - dst_buffer); // bytes written +} + +size_t lzfse_decode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size, + const uint8_t *__restrict src_buffer, + size_t src_size, void *__restrict scratch_buffer) { + int has_malloc = 0; + size_t ret = 0; + + // Deal with the possible NULL pointer + if (scratch_buffer == NULL) { + // +1 in case scratch size could be zero + scratch_buffer = malloc(lzfse_decode_scratch_size() + 1); + has_malloc = 1; + } + if (scratch_buffer == NULL) + return 0; + ret = lzfse_decode_buffer_with_scratch(dst_buffer, + dst_size, src_buffer, + src_size, scratch_buffer); + if (has_malloc) + free(scratch_buffer); + return ret; +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_decode_base.c b/view/kernelcache/core/transformers/liblzfse/lzfse_decode_base.c new file mode 100644 index 00000000..31fe8591 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_decode_base.c @@ -0,0 +1,630 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "lzfse_internal.h" +#include "lzvn_decode_base.h" + +/*! @abstract Decode an entry value from next bits of stream. + * Return \p value, and set \p *nbits to the number of bits to consume + * (starting with LSB). */ +static inline int lzfse_decode_v1_freq_value(uint32_t bits, int *nbits) { + static const int8_t lzfse_freq_nbits_table[32] = { + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14, + 2, 3, 2, 5, 2, 3, 2, 8, 2, 3, 2, 5, 2, 3, 2, 14}; + static const int8_t lzfse_freq_value_table[32] = { + 0, 2, 1, 4, 0, 3, 1, -1, 0, 2, 1, 5, 0, 3, 1, -1, + 0, 2, 1, 6, 0, 3, 1, -1, 0, 2, 1, 7, 0, 3, 1, -1}; + + uint32_t b = bits & 31; // lower 5 bits + int n = lzfse_freq_nbits_table[b]; + *nbits = n; + + // Special cases for > 5 bits encoding + if (n == 8) + return 8 + ((bits >> 4) & 0xf); + if (n == 14) + return 24 + ((bits >> 4) & 0x3ff); + + // <= 5 bits encoding from table + return lzfse_freq_value_table[b]; +} + +/*! @abstract Extracts up to 32 bits from a 64-bit field beginning at + * \p offset, and zero-extends them to a \p uint32_t. + * + * If we number the bits of \p v from 0 (least significant) to 63 (most + * significant), the result is bits \p offset to \p offset+nbits-1. */ +static inline uint32_t get_field(uint64_t v, int offset, int nbits) { + assert(offset + nbits < 64 && offset >= 0 && nbits <= 32); + if (nbits == 32) + return (uint32_t)(v >> offset); + return (uint32_t)((v >> offset) & ((1 << nbits) - 1)); +} + +/*! @abstract Return \c header_size field from a \c lzfse_compressed_block_header_v2. */ +static inline uint32_t +lzfse_decode_v2_header_size(const lzfse_compressed_block_header_v2 *in) { + return get_field(in->packed_fields[2], 0, 32); +} + +/*! @abstract Decode all fields from a \c lzfse_compressed_block_header_v2 to a + * \c lzfse_compressed_block_header_v1. + * @return 0 on success. + * @return -1 on failure. */ +static inline int lzfse_decode_v1(lzfse_compressed_block_header_v1 *out, + const lzfse_compressed_block_header_v2 *in) { + // Clear all fields + memset(out, 0x00, sizeof(lzfse_compressed_block_header_v1)); + + uint64_t v0 = in->packed_fields[0]; + uint64_t v1 = in->packed_fields[1]; + uint64_t v2 = in->packed_fields[2]; + + out->magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC; + out->n_raw_bytes = in->n_raw_bytes; + + // Literal state + out->n_literals = get_field(v0, 0, 20); + out->n_literal_payload_bytes = get_field(v0, 20, 20); + out->literal_bits = (int)get_field(v0, 60, 3) - 7; + out->literal_state[0] = get_field(v1, 0, 10); + out->literal_state[1] = get_field(v1, 10, 10); + out->literal_state[2] = get_field(v1, 20, 10); + out->literal_state[3] = get_field(v1, 30, 10); + + // L,M,D state + out->n_matches = get_field(v0, 40, 20); + out->n_lmd_payload_bytes = get_field(v1, 40, 20); + out->lmd_bits = (int)get_field(v1, 60, 3) - 7; + out->l_state = get_field(v2, 32, 10); + out->m_state = get_field(v2, 42, 10); + out->d_state = get_field(v2, 52, 10); + + // Total payload size + out->n_payload_bytes = + out->n_literal_payload_bytes + out->n_lmd_payload_bytes; + + // Freq tables + uint16_t *dst = &(out->l_freq[0]); + const uint8_t *src = &(in->freq[0]); + const uint8_t *src_end = + (const uint8_t *)in + get_field(v2, 0, 32); // first byte after header + uint32_t accum = 0; + int accum_nbits = 0; + + // No freq tables? + if (src_end == src) + return 0; // OK, freq tables were omitted + + for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS + + LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS; + i++) { + // Refill accum, one byte at a time, until we reach end of header, or accum + // is full + while (src < src_end && accum_nbits + 8 <= 32) { + accum |= (uint32_t)(*src) << accum_nbits; + accum_nbits += 8; + src++; + } + + // Decode and store value + int nbits = 0; + dst[i] = lzfse_decode_v1_freq_value(accum, &nbits); + + if (nbits > accum_nbits) + return -1; // failed + + // Consume nbits bits + accum >>= nbits; + accum_nbits -= nbits; + } + + if (accum_nbits >= 8 || src != src_end) + return -1; // we need to end up exactly at the end of header, with less than + // 8 bits in accumulator + + return 0; +} + +static inline void copy(uint8_t *dst, const uint8_t *src, size_t length) { + const uint8_t *dst_end = dst + length; + do { + copy8(dst, src); + dst += 8; + src += 8; + } while (dst < dst_end); +} + +static int lzfse_decode_lmd(lzfse_decoder_state *s) { + lzfse_compressed_block_decoder_state *bs = &(s->compressed_lzfse_block_state); + fse_state l_state = bs->l_state; + fse_state m_state = bs->m_state; + fse_state d_state = bs->d_state; + fse_in_stream in = bs->lmd_in_stream; + const uint8_t *src_start = s->src_begin; + const uint8_t *src = s->src + bs->lmd_in_buf; + const uint8_t *lit = bs->current_literal; + uint8_t *dst = s->dst; + uint32_t symbols = bs->n_matches; + int32_t L = bs->l_value; + int32_t M = bs->m_value; + int32_t D = bs->d_value; + + assert(l_state < LZFSE_ENCODE_L_STATES); + assert(m_state < LZFSE_ENCODE_M_STATES); + assert(d_state < LZFSE_ENCODE_D_STATES); + + // Number of bytes remaining in the destination buffer, minus 32 to + // provide a margin of safety for using overlarge copies on the fast path. + // This is a signed quantity, and may go negative when we are close to the + // end of the buffer. That's OK; we're careful about how we handle it + // in the slow-and-careful match execution path. + ptrdiff_t remaining_bytes = s->dst_end - dst - 32; + + // If L or M is non-zero, that means that we have already started decoding + // this block, and that we needed to interrupt decoding to get more space + // from the caller. There's a pending L, M, D triplet that we weren't + // able to completely process. Jump ahead to finish executing that symbol + // before decoding new values. + if (L || M) + goto ExecuteMatch; + + while (symbols > 0) { + int res; + // Decode the next L, M, D symbol from the input stream. + res = fse_in_flush(&in, &src, src_start); + if (res) { + return LZFSE_STATUS_ERROR; + } + L = fse_value_decode(&l_state, bs->l_decoder, &in); + assert(l_state < LZFSE_ENCODE_L_STATES); + if ((lit + L) >= (bs->literals + LZFSE_LITERALS_PER_BLOCK + 64)) { + return LZFSE_STATUS_ERROR; + } + res = fse_in_flush2(&in, &src, src_start); + if (res) { + return LZFSE_STATUS_ERROR; + } + M = fse_value_decode(&m_state, bs->m_decoder, &in); + assert(m_state < LZFSE_ENCODE_M_STATES); + res = fse_in_flush2(&in, &src, src_start); + if (res) { + return LZFSE_STATUS_ERROR; + } + int32_t new_d = fse_value_decode(&d_state, bs->d_decoder, &in); + assert(d_state < LZFSE_ENCODE_D_STATES); + D = new_d ? new_d : D; + symbols--; + + ExecuteMatch: + // Error if D is out of range, so that we avoid passing through + // uninitialized data or accesssing memory out of the destination + // buffer. + if ((uint32_t)D > dst + L - s->dst_begin) + return LZFSE_STATUS_ERROR; + + if (L + M <= remaining_bytes) { + // If we have plenty of space remaining, we can copy the literal + // and match with 16- and 32-byte operations, without worrying + // about writing off the end of the buffer. + remaining_bytes -= L + M; + copy(dst, lit, L); + dst += L; + lit += L; + // For the match, we have two paths; a fast copy by 16-bytes if + // the match distance is large enough to allow it, and a more + // careful path that applies a permutation to account for the + // possible overlap between source and destination if the distance + // is small. + if (D >= 8 || D >= M) + copy(dst, dst - D, M); + else + for (size_t i = 0; i < M; i++) + dst[i] = dst[i - D]; + dst += M; + } + + else { + // Otherwise, we are very close to the end of the destination + // buffer, so we cannot use wide copies that slop off the end + // of the region that we are copying to. First, we restore + // the true length remaining, rather than the sham value we've + // been using so far. + remaining_bytes += 32; + // Now, we process the literal. Either there's space for it + // or there isn't; if there is, we copy the whole thing and + // update all the pointers and lengths to reflect the copy. + if (L <= remaining_bytes) { + for (size_t i = 0; i < L; i++) + dst[i] = lit[i]; + dst += L; + lit += L; + remaining_bytes -= L; + L = 0; + } + // There isn't enough space to fit the whole literal. Copy as + // much of it as we can, update the pointers and the value of + // L, and report that the destination buffer is full. Note that + // we always write right up to the end of the destination buffer. + else { + for (size_t i = 0; i < remaining_bytes; i++) + dst[i] = lit[i]; + dst += remaining_bytes; + lit += remaining_bytes; + L -= remaining_bytes; + goto DestinationBufferIsFull; + } + // The match goes just like the literal does. We copy as much as + // we can byte-by-byte, and if we reach the end of the buffer + // before finishing, we return to the caller indicating that + // the buffer is full. + if (M <= remaining_bytes) { + for (size_t i = 0; i < M; i++) + dst[i] = dst[i - D]; + dst += M; + remaining_bytes -= M; + M = 0; + (void)M; // no dead store warning + // We don't need to update M = 0, because there's no partial + // symbol to continue executing. Either we're at the end of + // the block, in which case we will never need to resume with + // this state, or we're going to decode another L, M, D set, + // which will overwrite M anyway. + // + // But we still set M = 0, to maintain the post-condition. + } else { + for (size_t i = 0; i < remaining_bytes; i++) + dst[i] = dst[i - D]; + dst += remaining_bytes; + M -= remaining_bytes; + DestinationBufferIsFull: + // Because we want to be able to resume decoding where we've left + // off (even in the middle of a literal or match), we need to + // update all of the block state fields with the current values + // so that we can resume execution from this point once the + // caller has given us more space to write into. + bs->l_value = L; + bs->m_value = M; + bs->d_value = D; + bs->l_state = l_state; + bs->m_state = m_state; + bs->d_state = d_state; + bs->lmd_in_stream = in; + bs->n_matches = symbols; + bs->lmd_in_buf = (uint32_t)(src - s->src); + bs->current_literal = lit; + s->dst = dst; + return LZFSE_STATUS_DST_FULL; + } + // Restore the "sham" decremented value of remaining_bytes and + // continue to the next L, M, D triple. We'll just be back in + // the careful path again, but this only happens at the very end + // of the buffer, so a little minor inefficiency here is a good + // tradeoff for simpler code. + remaining_bytes -= 32; + } + } + // Because we've finished with the whole block, we don't need to update + // any of the blockstate fields; they will not be used again. We just + // update the destination pointer in the state object and return. + s->dst = dst; + return LZFSE_STATUS_OK; +} + +int lzfse_decode(lzfse_decoder_state *s) { + while (1) { + // Are we inside a block? + switch (s->block_magic) { + case LZFSE_NO_BLOCK_MAGIC: { + // We need at least 4 bytes of magic number to identify next block + if (s->src + 4 > s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated + uint32_t magic = load4(s->src); + + if (magic == LZFSE_ENDOFSTREAM_BLOCK_MAGIC) { + s->src += 4; + s->end_of_stream = 1; + return LZFSE_STATUS_OK; // done + } + + if (magic == LZFSE_UNCOMPRESSED_BLOCK_MAGIC) { + if (s->src + sizeof(uncompressed_block_header) > s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated + // Setup state for uncompressed block + uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state); + bs->n_raw_bytes = + load4(s->src + offsetof(uncompressed_block_header, n_raw_bytes)); + s->src += sizeof(uncompressed_block_header); + s->block_magic = magic; + break; + } + + if (magic == LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC) { + if (s->src + sizeof(lzvn_compressed_block_header) > s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated + // Setup state for compressed LZVN block + lzvn_compressed_block_decoder_state *bs = + &(s->compressed_lzvn_block_state); + bs->n_raw_bytes = + load4(s->src + offsetof(lzvn_compressed_block_header, n_raw_bytes)); + bs->n_payload_bytes = load4( + s->src + offsetof(lzvn_compressed_block_header, n_payload_bytes)); + bs->d_prev = 0; + s->src += sizeof(lzvn_compressed_block_header); + s->block_magic = magic; + break; + } + + if (magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC || + magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) { + lzfse_compressed_block_header_v1 header1; + size_t header_size = 0; + + // Decode compressed headers + if (magic == LZFSE_COMPRESSEDV2_BLOCK_MAGIC) { + // Check we have the fixed part of the structure + if (s->src + offsetof(lzfse_compressed_block_header_v2, freq) > s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated + + // Get size, and check we have the entire structure + const lzfse_compressed_block_header_v2 *header2 = + (const lzfse_compressed_block_header_v2 *)s->src; // not aligned, OK + header_size = lzfse_decode_v2_header_size(header2); + if (s->src + header_size > s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated + int decodeStatus = lzfse_decode_v1(&header1, header2); + if (decodeStatus != 0) + return LZFSE_STATUS_ERROR; // failed + } else { + if (s->src + sizeof(lzfse_compressed_block_header_v1) > s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // SRC truncated + memcpy(&header1, s->src, sizeof(lzfse_compressed_block_header_v1)); + header_size = sizeof(lzfse_compressed_block_header_v1); + } + + // We require the header + entire encoded block to be present in SRC + // during the entire block decoding. + // This can be relaxed somehow, if it becomes a limiting factor, at the + // price of a more complex state maintenance. + // For DST, we can't easily require space for the entire decoded block, + // because it may expand to something very very large. + if (s->src + header_size + header1.n_literal_payload_bytes + + header1.n_lmd_payload_bytes > + s->src_end) + return LZFSE_STATUS_SRC_EMPTY; // need all encoded block + + // Sanity checks + if (lzfse_check_block_header_v1(&header1) != 0) { + return LZFSE_STATUS_ERROR; + } + + // Skip header + s->src += header_size; + + // Setup state for compressed V1 block from header + lzfse_compressed_block_decoder_state *bs = + &(s->compressed_lzfse_block_state); + bs->n_lmd_payload_bytes = header1.n_lmd_payload_bytes; + bs->n_matches = header1.n_matches; + fse_init_decoder_table(LZFSE_ENCODE_LITERAL_STATES, + LZFSE_ENCODE_LITERAL_SYMBOLS, + header1.literal_freq, bs->literal_decoder); + fse_init_value_decoder_table( + LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, header1.l_freq, + l_extra_bits, l_base_value, bs->l_decoder); + fse_init_value_decoder_table( + LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, header1.m_freq, + m_extra_bits, m_base_value, bs->m_decoder); + fse_init_value_decoder_table( + LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, header1.d_freq, + d_extra_bits, d_base_value, bs->d_decoder); + + // Decode literals + { + fse_in_stream in; + const uint8_t *buf_start = s->src_begin; + s->src += header1.n_literal_payload_bytes; // skip literal payload + const uint8_t *buf = s->src; // read bits backwards from the end + if (fse_in_init(&in, header1.literal_bits, &buf, buf_start) != 0) + return LZFSE_STATUS_ERROR; + + fse_state state0 = header1.literal_state[0]; + fse_state state1 = header1.literal_state[1]; + fse_state state2 = header1.literal_state[2]; + fse_state state3 = header1.literal_state[3]; + + for (uint32_t i = 0; i < header1.n_literals; i += 4) // n_literals is multiple of 4 + { +#if FSE_IOSTREAM_64 + if (fse_in_flush(&in, &buf, buf_start) != 0) + return LZFSE_STATUS_ERROR; // [57, 64] bits + bs->literals[i + 0] = + fse_decode(&state0, bs->literal_decoder, &in); // 10b max + bs->literals[i + 1] = + fse_decode(&state1, bs->literal_decoder, &in); // 10b max + bs->literals[i + 2] = + fse_decode(&state2, bs->literal_decoder, &in); // 10b max + bs->literals[i + 3] = + fse_decode(&state3, bs->literal_decoder, &in); // 10b max +#else + if (fse_in_flush(&in, &buf, buf_start) != 0) + return LZFSE_STATUS_ERROR; // [25, 23] bits + bs->literals[i + 0] = + fse_decode(&state0, bs->literal_decoder, &in); // 10b max + bs->literals[i + 1] = + fse_decode(&state1, bs->literal_decoder, &in); // 10b max + if (fse_in_flush(&in, &buf, buf_start) != 0) + return LZFSE_STATUS_ERROR; // [25, 23] bits + bs->literals[i + 2] = + fse_decode(&state2, bs->literal_decoder, &in); // 10b max + bs->literals[i + 3] = + fse_decode(&state3, bs->literal_decoder, &in); // 10b max +#endif + } + + bs->current_literal = bs->literals; + } // literals + + // SRC is not incremented to skip the LMD payload, since we need it + // during block decode. + // We will increment SRC at the end of the block only after this point. + + // Initialize the L,M,D decode stream, do not start decoding matches + // yet, and store decoder state + { + fse_in_stream in; + // read bits backwards from the end + const uint8_t *buf = s->src + header1.n_lmd_payload_bytes; + if (fse_in_init(&in, header1.lmd_bits, &buf, s->src) != 0) + return LZFSE_STATUS_ERROR; + + bs->l_state = header1.l_state; + bs->m_state = header1.m_state; + bs->d_state = header1.d_state; + bs->lmd_in_buf = (uint32_t)(buf - s->src); + bs->l_value = bs->m_value = 0; + // Initialize D to an illegal value so we can't erroneously use + // an uninitialized "previous" value. + bs->d_value = -1; + bs->lmd_in_stream = in; + } + + s->block_magic = magic; + break; + } + + // Here we have an invalid magic number + return LZFSE_STATUS_ERROR; + } // LZFSE_NO_BLOCK_MAGIC + + case LZFSE_UNCOMPRESSED_BLOCK_MAGIC: { + uncompressed_block_decoder_state *bs = &(s->uncompressed_block_state); + + // Compute the size (in bytes) of the data that we will actually copy. + // This size is minimum(bs->n_raw_bytes, space in src, space in dst). + + uint32_t copy_size = bs->n_raw_bytes; // bytes left to copy + if (copy_size == 0) { + s->block_magic = 0; + break; + } // end of block + + if (s->src_end <= s->src) + return LZFSE_STATUS_SRC_EMPTY; // need more SRC data + const size_t src_space = s->src_end - s->src; + if (copy_size > src_space) + copy_size = (uint32_t)src_space; // limit to SRC data (> 0) + + if (s->dst_end <= s->dst) + return LZFSE_STATUS_DST_FULL; // need more DST capacity + const size_t dst_space = s->dst_end - s->dst; + if (copy_size > dst_space) + copy_size = (uint32_t)dst_space; // limit to DST capacity (> 0) + + // Now that we know that the copy size is bounded to the source and + // dest buffers, go ahead and copy the data. + // We always have copy_size > 0 here + memcpy(s->dst, s->src, copy_size); + s->src += copy_size; + s->dst += copy_size; + bs->n_raw_bytes -= copy_size; + + break; + } // LZFSE_UNCOMPRESSED_BLOCK_MAGIC + + case LZFSE_COMPRESSEDV1_BLOCK_MAGIC: + case LZFSE_COMPRESSEDV2_BLOCK_MAGIC: { + lzfse_compressed_block_decoder_state *bs = + &(s->compressed_lzfse_block_state); + // Require the entire LMD payload to be in SRC + if (s->src_end <= s->src || + bs->n_lmd_payload_bytes > (size_t)(s->src_end - s->src)) + return LZFSE_STATUS_SRC_EMPTY; + + int status = lzfse_decode_lmd(s); + if (status != LZFSE_STATUS_OK) + return status; + + s->block_magic = LZFSE_NO_BLOCK_MAGIC; + s->src += bs->n_lmd_payload_bytes; // to next block + break; + } // LZFSE_COMPRESSEDV1_BLOCK_MAGIC || LZFSE_COMPRESSEDV2_BLOCK_MAGIC + + case LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC: { + lzvn_compressed_block_decoder_state *bs = + &(s->compressed_lzvn_block_state); + if (bs->n_payload_bytes > 0 && s->src_end <= s->src) + return LZFSE_STATUS_SRC_EMPTY; // need more SRC data + + // Init LZVN decoder state + lzvn_decoder_state dstate; + memset(&dstate, 0x00, sizeof(dstate)); + dstate.src = s->src; + dstate.src_end = s->src_end; + if (dstate.src_end - s->src > bs->n_payload_bytes) + dstate.src_end = s->src + bs->n_payload_bytes; // limit to payload bytes + dstate.dst_begin = s->dst_begin; + dstate.dst = s->dst; + dstate.dst_end = s->dst_end; + if (dstate.dst_end - s->dst > bs->n_raw_bytes) + dstate.dst_end = s->dst + bs->n_raw_bytes; // limit to raw bytes + dstate.d_prev = bs->d_prev; + dstate.end_of_stream = 0; + + // Run LZVN decoder + lzvn_decode(&dstate); + + // Update our state + size_t src_used = dstate.src - s->src; + size_t dst_used = dstate.dst - s->dst; + if (src_used > bs->n_payload_bytes || dst_used > bs->n_raw_bytes) + return LZFSE_STATUS_ERROR; // sanity check + s->src = dstate.src; + s->dst = dstate.dst; + bs->n_payload_bytes -= (uint32_t)src_used; + bs->n_raw_bytes -= (uint32_t)dst_used; + bs->d_prev = (uint32_t)dstate.d_prev; + + // Test end of block + if (bs->n_payload_bytes == 0 && bs->n_raw_bytes == 0 && + dstate.end_of_stream) { + s->block_magic = 0; + break; + } // block done + + // Check for invalid state + if (bs->n_payload_bytes == 0 || bs->n_raw_bytes == 0 || + dstate.end_of_stream) + return LZFSE_STATUS_ERROR; + + // Here, block is not done and state is valid, so we need more space in dst. + return LZFSE_STATUS_DST_FULL; + } + + default: + return LZFSE_STATUS_ERROR; // invalid magic + + } // switch magic + + } // block loop + + return LZFSE_STATUS_OK; +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_encode.c b/view/kernelcache/core/transformers/liblzfse/lzfse_encode.c new file mode 100644 index 00000000..4819a0f9 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_encode.c @@ -0,0 +1,163 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZFSE encode API + +#include "lzfse.h" +#include "lzfse_internal.h" + +size_t lzfse_encode_scratch_size() { + size_t s1 = sizeof(lzfse_encoder_state); + size_t s2 = lzvn_encode_scratch_size(); + return (s1 > s2) ? s1 : s2; // max(lzfse,lzvn) +} + +size_t lzfse_encode_buffer_with_scratch(uint8_t *__restrict dst_buffer, + size_t dst_size, const uint8_t *__restrict src_buffer, + size_t src_size, void *__restrict scratch_buffer) { + const size_t original_size = src_size; + + // If input is really really small, go directly to uncompressed buffer + // (because LZVN will refuse to encode it, and we will report a failure) + if (src_size < LZVN_ENCODE_MIN_SRC_SIZE) + goto try_uncompressed; + + // If input is too small, try encoding with LZVN + if (src_size < LZFSE_ENCODE_LZVN_THRESHOLD) { + // need header + end-of-stream marker + size_t extra_size = 4 + sizeof(lzvn_compressed_block_header); + if (dst_size <= extra_size) + goto try_uncompressed; // DST is really too small, give up + + size_t sz = lzvn_encode_buffer( + dst_buffer + sizeof(lzvn_compressed_block_header), + dst_size - extra_size, src_buffer, src_size, scratch_buffer); + if (sz == 0 || sz >= src_size) + goto try_uncompressed; // failed, or no compression, fall back to + // uncompressed block + + // If we could encode, setup header and end-of-stream marker (we left room + // for them, no need to test) + lzvn_compressed_block_header header; + header.magic = LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC; + header.n_raw_bytes = (uint32_t)src_size; + header.n_payload_bytes = (uint32_t)sz; + memcpy(dst_buffer, &header, sizeof(header)); + store4(dst_buffer + sizeof(lzvn_compressed_block_header) + sz, + LZFSE_ENDOFSTREAM_BLOCK_MAGIC); + + return sz + extra_size; + } + + // Try encoding with LZFSE + { + lzfse_encoder_state *state = scratch_buffer; + memset(state, 0x00, sizeof *state); + if (lzfse_encode_init(state) != LZFSE_STATUS_OK) + goto try_uncompressed; + state->dst = dst_buffer; + state->dst_begin = dst_buffer; + state->dst_end = &dst_buffer[dst_size]; + state->src = src_buffer; + state->src_encode_i = 0; + + if (src_size >= 0xffffffffU) { + // lzfse only uses 32 bits for offsets internally, so if the input + // buffer is really huge, we need to process it in smaller chunks. + // Note that we switch over to this path for sizes much smaller + // 2GB because it's actually faster to change algorithms well before + // it's necessary for correctness. + // The first chunk, we just process normally. + const lzfse_offset encoder_block_size = 262144; + state->src_end = encoder_block_size; + if (lzfse_encode_base(state) != LZFSE_STATUS_OK) + goto try_uncompressed; + src_size -= encoder_block_size; + while (src_size >= encoder_block_size) { + // All subsequent chunks require a translation to keep the offsets + // from getting too big. Note that we are always going from + // encoder_block_size up to 2*encoder_block_size so that the + // offsets remain positive (as opposed to resetting to zero and + // having negative offsets). + state->src_end = 2 * encoder_block_size; + if (lzfse_encode_base(state) != LZFSE_STATUS_OK) + goto try_uncompressed; + lzfse_encode_translate(state, encoder_block_size); + src_size -= encoder_block_size; + } + // Set the end for the final chunk. + state->src_end = encoder_block_size + (lzfse_offset)src_size; + } + // If the source buffer is small enough to use 32-bit offsets, we simply + // encode the whole thing in a single chunk. + else + state->src_end = (lzfse_offset)src_size; + // This is either the trailing chunk (if the source file is huge), or + // the whole source file. + if (lzfse_encode_base(state) != LZFSE_STATUS_OK) + goto try_uncompressed; + if (lzfse_encode_finish(state) != LZFSE_STATUS_OK) + goto try_uncompressed; + // No error occured, return compressed size. + return state->dst - dst_buffer; + } + +try_uncompressed: + // Compression failed for some reason. If we can fit the data into the + // output buffer uncompressed, go ahead and do that instead. + if (original_size + 12 <= dst_size && original_size < INT32_MAX) { + uncompressed_block_header header = {.magic = LZFSE_UNCOMPRESSED_BLOCK_MAGIC, + .n_raw_bytes = (uint32_t)src_size}; + uint8_t *dst_end = dst_buffer; + memcpy(dst_end, &header, sizeof header); + dst_end += sizeof header; + memcpy(dst_end, src_buffer, original_size); + dst_end += original_size; + store4(dst_end, LZFSE_ENDOFSTREAM_BLOCK_MAGIC); + dst_end += 4; + return dst_end - dst_buffer; + } + + // Otherwise, there's nothing we can do, so return zero. + return 0; +} + +size_t lzfse_encode_buffer(uint8_t *__restrict dst_buffer, size_t dst_size, + const uint8_t *__restrict src_buffer, + size_t src_size, void *__restrict scratch_buffer) { + int has_malloc = 0; + size_t ret = 0; + + // Deal with the possible NULL pointer + if (scratch_buffer == NULL) { + // +1 in case scratch size could be zero + scratch_buffer = malloc(lzfse_encode_scratch_size() + 1); + has_malloc = 1; + } + if (scratch_buffer == NULL) + return 0; + ret = lzfse_encode_buffer_with_scratch(dst_buffer, + dst_size, src_buffer, + src_size, scratch_buffer); + if (has_malloc) + free(scratch_buffer); + return ret; +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_encode_base.c b/view/kernelcache/core/transformers/liblzfse/lzfse_encode_base.c new file mode 100644 index 00000000..367d7a23 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_encode_base.c @@ -0,0 +1,830 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZFSE encoder + +#include "lzfse_internal.h" +#include "lzfse_encode_tables.h" + +/*! @abstract Get hash in range [0, LZFSE_ENCODE_HASH_VALUES-1] from 4 bytes in X. */ +static inline uint32_t hashX(uint32_t x) { + return (x * 2654435761U) >> + (32 - LZFSE_ENCODE_HASH_BITS); // Knuth multiplicative hash +} + +/*! @abstract Return value with all 0 except nbits<=32 unsigned bits from V + * at bit offset OFFSET. + * V is assumed to fit on nbits bits. */ +static inline uint64_t setField(uint32_t v, int offset, int nbits) { + assert(offset + nbits < 64 && offset >= 0 && nbits <= 32); + assert(nbits == 32 || (v < (1U << nbits))); + return ((uint64_t)v << (uint64_t)offset); +} + +/*! @abstract Encode all fields, except freq, from a + * lzfse_compressed_block_header_v1 to a lzfse_compressed_block_header_v2. + * All but the header_size and freq fields of the output are modified. */ +static inline void +lzfse_encode_v1_state(lzfse_compressed_block_header_v2 *out, + const lzfse_compressed_block_header_v1 *in) { + out->magic = LZFSE_COMPRESSEDV2_BLOCK_MAGIC; + out->n_raw_bytes = in->n_raw_bytes; + + // Literal state + out->packed_fields[0] = setField(in->n_literals, 0, 20) | + setField(in->n_literal_payload_bytes, 20, 20) | + setField(in->n_matches, 40, 20) | + setField(7 + in->literal_bits, 60, 3); + out->packed_fields[1] = setField(in->literal_state[0], 0, 10) | + setField(in->literal_state[1], 10, 10) | + setField(in->literal_state[2], 20, 10) | + setField(in->literal_state[3], 30, 10) | + setField(in->n_lmd_payload_bytes, 40, 20) | + setField(7 + in->lmd_bits, 60, 3); + out->packed_fields[2] = out->packed_fields[2] // header_size already stored in v[2] + | setField(in->l_state, 32, 10) | setField(in->m_state, 42, 10) | + setField(in->d_state, 52, 10); +} + +/*! @abstract Encode an entry value in a freq table. Return bits, and sets + * *nbits to the number of bits to serialize. */ +static inline uint32_t lzfse_encode_v1_freq_value(int value, int *nbits) { + // Fixed Huffman code, bits are read from LSB. + // Note that we rely on the position of the first '0' bit providing the number + // of bits. + switch (value) { + case 0: + *nbits = 2; + return 0; // 0.0 + case 1: + *nbits = 2; + return 2; // 1.0 + case 2: + *nbits = 3; + return 1; // 0.01 + case 3: + *nbits = 3; + return 5; // 1.01 + case 4: + *nbits = 5; + return 3; // 00.011 + case 5: + *nbits = 5; + return 11; // 01.011 + case 6: + *nbits = 5; + return 19; // 10.011 + case 7: + *nbits = 5; + return 27; // 11.011 + default: + break; + } + if (value < 24) { + *nbits = 8; // 4+4 + return 7 + ((value - 8) << 4); // xxxx.0111 + } + // 24..1047 + *nbits = 14; // 4+10 + return ((value - 24) << 4) + 15; // xxxxxxxxxx.1111 +} + +/*! @abstract Encode all tables from a lzfse_compressed_block_header_v1 + * to a lzfse_compressed_block_header_v2. + * Only the header_size and freq fields of the output are modified. + * @return Size of the lzfse_compressed_block_header_v2 */ +static inline size_t +lzfse_encode_v1_freq_table(lzfse_compressed_block_header_v2 *out, + const lzfse_compressed_block_header_v1 *in) { + uint32_t accum = 0; + int accum_nbits = 0; + const uint16_t *src = &(in->l_freq[0]); // first value of first table (struct + // will not be modified, so this code + // will remain valid) + uint8_t *dst = &(out->freq[0]); + for (int i = 0; i < LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS + + LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS; + i++) { + // Encode one value to accum + int nbits = 0; + uint32_t bits = lzfse_encode_v1_freq_value(src[i], &nbits); + assert(bits < (1 << nbits)); + accum |= bits << accum_nbits; + accum_nbits += nbits; + + // Store bytes from accum to output buffer + while (accum_nbits >= 8) { + *dst = (uint8_t)(accum & 0xff); + accum >>= 8; + accum_nbits -= 8; + dst++; + } + } + // Store final byte if needed + if (accum_nbits > 0) { + *dst = (uint8_t)(accum & 0xff); + dst++; + } + + // Return final size of out + uint32_t header_size = (uint32_t)(dst - (uint8_t *)out); + out->packed_fields[0] = 0; + out->packed_fields[1] = 0; + out->packed_fields[2] = setField(header_size, 0, 32); + + return header_size; +} + +// We need to limit forward match length to make sure it won't split into a too +// large number of LMD. +// The limit itself is quite large, so it doesn't really impact compression +// ratio. +// The matches may still be expanded backwards by a few bytes, so the final +// length may be greater than this limit, which is OK. +#define LZFSE_ENCODE_MAX_MATCH_LENGTH (100 * LZFSE_ENCODE_MAX_M_VALUE) + +// =============================================================== +// Encoder back end + +/*! @abstract Encode matches stored in STATE into a compressed/uncompressed block. + * @return LZFSE_STATUS_OK on success. + * @return LZFSE_STATUS_DST_FULL and restore initial state if output buffer is + * full. */ +static int lzfse_encode_matches(lzfse_encoder_state *s) { + if (s->n_literals == 0 && s->n_matches == 0) + return LZFSE_STATUS_OK; // nothing to store, OK + + uint32_t l_occ[LZFSE_ENCODE_L_SYMBOLS]; + uint32_t m_occ[LZFSE_ENCODE_M_SYMBOLS]; + uint32_t d_occ[LZFSE_ENCODE_D_SYMBOLS]; + uint32_t literal_occ[LZFSE_ENCODE_LITERAL_SYMBOLS]; + fse_encoder_entry l_encoder[LZFSE_ENCODE_L_SYMBOLS]; + fse_encoder_entry m_encoder[LZFSE_ENCODE_M_SYMBOLS]; + fse_encoder_entry d_encoder[LZFSE_ENCODE_D_SYMBOLS]; + fse_encoder_entry literal_encoder[LZFSE_ENCODE_LITERAL_SYMBOLS]; + int ok = 1; + lzfse_compressed_block_header_v1 header1 = {0}; + lzfse_compressed_block_header_v2 *header2 = 0; + + // Keep initial state to be able to restore it if DST full + uint8_t *dst0 = s->dst; + uint32_t n_literals0 = s->n_literals; + + // Add 0x00 literals until n_literals multiple of 4, since we encode 4 + // interleaved literal streams. + while (s->n_literals & 3) { + uint32_t n = s->n_literals++; + s->literals[n] = 0; + } + + // Encode previous distance + uint32_t d_prev = 0; + for (uint32_t i = 0; i < s->n_matches; i++) { + uint32_t d = s->d_values[i]; + if (d == d_prev) + s->d_values[i] = 0; + else + d_prev = d; + } + + // Clear occurrence tables + memset(l_occ, 0, sizeof(l_occ)); + memset(m_occ, 0, sizeof(m_occ)); + memset(d_occ, 0, sizeof(d_occ)); + memset(literal_occ, 0, sizeof(literal_occ)); + + // Update occurrence tables in all 4 streams (L,M,D,literals) + uint32_t l_sum = 0; + uint32_t m_sum = 0; + for (uint32_t i = 0; i < s->n_matches; i++) { + uint32_t l = s->l_values[i]; + l_sum += l; + l_occ[l_base_from_value(l)]++; + } + for (uint32_t i = 0; i < s->n_matches; i++) { + uint32_t m = s->m_values[i]; + m_sum += m; + m_occ[m_base_from_value(m)]++; + } + for (uint32_t i = 0; i < s->n_matches; i++) + d_occ[d_base_from_value(s->d_values[i])]++; + for (uint32_t i = 0; i < s->n_literals; i++) + literal_occ[s->literals[i]]++; + + // Make sure we have enough room for a _full_ V2 header + if (s->dst + sizeof(lzfse_compressed_block_header_v2) > s->dst_end) { + ok = 0; + goto END; + } + header2 = (lzfse_compressed_block_header_v2 *)(s->dst); + + // Setup header V1 + header1.magic = LZFSE_COMPRESSEDV1_BLOCK_MAGIC; + header1.n_raw_bytes = m_sum + l_sum; + header1.n_matches = s->n_matches; + header1.n_literals = s->n_literals; + + // Normalize occurrence tables to freq tables + fse_normalize_freq(LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, l_occ, + header1.l_freq); + fse_normalize_freq(LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, m_occ, + header1.m_freq); + fse_normalize_freq(LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, d_occ, + header1.d_freq); + fse_normalize_freq(LZFSE_ENCODE_LITERAL_STATES, LZFSE_ENCODE_LITERAL_SYMBOLS, + literal_occ, header1.literal_freq); + + // Compress freq tables to V2 header, and get actual size of V2 header + s->dst += lzfse_encode_v1_freq_table(header2, &header1); + + // Initialize encoder tables from freq tables + fse_init_encoder_table(LZFSE_ENCODE_L_STATES, LZFSE_ENCODE_L_SYMBOLS, + header1.l_freq, l_encoder); + fse_init_encoder_table(LZFSE_ENCODE_M_STATES, LZFSE_ENCODE_M_SYMBOLS, + header1.m_freq, m_encoder); + fse_init_encoder_table(LZFSE_ENCODE_D_STATES, LZFSE_ENCODE_D_SYMBOLS, + header1.d_freq, d_encoder); + fse_init_encoder_table(LZFSE_ENCODE_LITERAL_STATES, + LZFSE_ENCODE_LITERAL_SYMBOLS, header1.literal_freq, + literal_encoder); + + // Encode literals + { + fse_out_stream out; + fse_out_init(&out); + fse_state state0, state1, state2, state3; + state0 = state1 = state2 = state3 = 0; + + uint8_t *buf = s->dst; + uint32_t i = s->n_literals; // I multiple of 4 + // We encode starting from the last literal so we can decode starting from + // the first + while (i > 0) { + if (buf + 16 > s->dst_end) { + ok = 0; + goto END; + } // out full + i -= 4; + fse_encode(&state3, literal_encoder, &out, s->literals[i + 3]); // 10b + fse_encode(&state2, literal_encoder, &out, s->literals[i + 2]); // 10b +#if !FSE_IOSTREAM_64 + fse_out_flush(&out, &buf); +#endif + fse_encode(&state1, literal_encoder, &out, s->literals[i + 1]); // 10b + fse_encode(&state0, literal_encoder, &out, s->literals[i + 0]); // 10b + fse_out_flush(&out, &buf); + } + fse_out_finish(&out, &buf); + + // Update header with final encoder state + header1.literal_bits = out.accum_nbits; // [-7, 0] + header1.n_literal_payload_bytes = (uint32_t)(buf - s->dst); + header1.literal_state[0] = state0; + header1.literal_state[1] = state1; + header1.literal_state[2] = state2; + header1.literal_state[3] = state3; + + // Update state + s->dst = buf; + + } // literals + + // Encode L,M,D + { + fse_out_stream out; + fse_out_init(&out); + fse_state l_state, m_state, d_state; + l_state = m_state = d_state = 0; + + uint8_t *buf = s->dst; + uint32_t i = s->n_matches; + + // Add 8 padding bytes to the L,M,D payload + if (buf + 8 > s->dst_end) { + ok = 0; + goto END; + } // out full + store8(buf, 0); + buf += 8; + + // We encode starting from the last match so we can decode starting from the + // first + while (i > 0) { + if (buf + 16 > s->dst_end) { + ok = 0; + goto END; + } // out full + i -= 1; + + // D requires 23b max + int32_t d_value = s->d_values[i]; + uint8_t d_symbol = d_base_from_value(d_value); + int32_t d_nbits = d_extra_bits[d_symbol]; + int32_t d_bits = d_value - d_base_value[d_symbol]; + fse_out_push(&out, d_nbits, d_bits); + fse_encode(&d_state, d_encoder, &out, d_symbol); +#if !FSE_IOSTREAM_64 + fse_out_flush(&out, &buf); +#endif + + // M requires 17b max + int32_t m_value = s->m_values[i]; + uint8_t m_symbol = m_base_from_value(m_value); + int32_t m_nbits = m_extra_bits[m_symbol]; + int32_t m_bits = m_value - m_base_value[m_symbol]; + fse_out_push(&out, m_nbits, m_bits); + fse_encode(&m_state, m_encoder, &out, m_symbol); +#if !FSE_IOSTREAM_64 + fse_out_flush(&out, &buf); +#endif + + // L requires 14b max + int32_t l_value = s->l_values[i]; + uint8_t l_symbol = l_base_from_value(l_value); + int32_t l_nbits = l_extra_bits[l_symbol]; + int32_t l_bits = l_value - l_base_value[l_symbol]; + fse_out_push(&out, l_nbits, l_bits); + fse_encode(&l_state, l_encoder, &out, l_symbol); + fse_out_flush(&out, &buf); + } + fse_out_finish(&out, &buf); + + // Update header with final encoder state + header1.n_lmd_payload_bytes = (uint32_t)(buf - s->dst); + header1.lmd_bits = out.accum_nbits; // [-7, 0] + header1.l_state = l_state; + header1.m_state = m_state; + header1.d_state = d_state; + + // Update state + s->dst = buf; + + } // L,M,D + + // Final state update, here we had enough space in DST, and are not going to + // revert state + s->n_literals = 0; + s->n_matches = 0; + + // Final payload size + header1.n_payload_bytes = + header1.n_literal_payload_bytes + header1.n_lmd_payload_bytes; + + // Encode state info in V2 header (we previously encoded the tables, now we + // set the other fields) + lzfse_encode_v1_state(header2, &header1); + +END: + if (!ok) { + // Revert state, DST was full + + // Revert the d_prev encoding + uint32_t d_prev = 0; + for (uint32_t i = 0; i < s->n_matches; i++) { + uint32_t d = s->d_values[i]; + if (d == 0) + s->d_values[i] = d_prev; + else + d_prev = d; + } + + // Revert literal count + s->n_literals = n_literals0; + + // Revert DST + s->dst = dst0; + + return LZFSE_STATUS_DST_FULL; // DST full + } + + return LZFSE_STATUS_OK; +} + +/*! @abstract Push a L,M,D match into the STATE. + * @return LZFSE_STATUS_OK if OK. + * @return LZFSE_STATUS_DST_FULL if the match can't be pushed, meaning one of + * the buffers is full. In that case the state is not modified. */ +static inline int lzfse_push_lmd(lzfse_encoder_state *s, uint32_t L, + uint32_t M, uint32_t D) { + // Check if we have enough space to push the match (we add some margin to copy + // literals faster here, and round final count later) + if (s->n_matches + 1 + 8 > LZFSE_MATCHES_PER_BLOCK) + return LZFSE_STATUS_DST_FULL; // state full + if (s->n_literals + L + 16 > LZFSE_LITERALS_PER_BLOCK) + return LZFSE_STATUS_DST_FULL; // state full + + // Store match + uint32_t n = s->n_matches++; + s->l_values[n] = L; + s->m_values[n] = M; + s->d_values[n] = D; + + // Store literals + uint8_t *dst = s->literals + s->n_literals; + const uint8_t *src = s->src + s->src_literal; + uint8_t *dst_end = dst + L; + if (s->src_literal + L + 16 > s->src_end) { + // Careful at the end of SRC, we can't read 16 bytes + if (L > 0) + memcpy(dst, src, L); + } else { + copy16(dst, src); + dst += 16; + src += 16; + while (dst < dst_end) { + copy16(dst, src); + dst += 16; + src += 16; + } + } + s->n_literals += L; + + // Update state + s->src_literal += L + M; + + return LZFSE_STATUS_OK; +} + +/*! @abstract Split MATCH into one or more L,M,D parts, and push to STATE. + * @return LZFSE_STATUS_OK if OK. + * @return LZFSE_STATUS_DST_FULL if the match can't be pushed, meaning one of the + * buffers is full. In that case the state is not modified. */ +static int lzfse_push_match(lzfse_encoder_state *s, const lzfse_match *match) { + // Save the initial n_matches, n_literals, src_literal + uint32_t n_matches0 = s->n_matches; + uint32_t n_literals0 = s->n_literals; + lzfse_offset src_literals0 = s->src_literal; + + // L,M,D + uint32_t L = (uint32_t)(match->pos - s->src_literal); // literal count + uint32_t M = match->length; // match length + uint32_t D = (uint32_t)(match->pos - match->ref); // match distance + int ok = 1; + + // Split L if too large + while (L > LZFSE_ENCODE_MAX_L_VALUE) { + if (lzfse_push_lmd(s, LZFSE_ENCODE_MAX_L_VALUE, 0, 1) != 0) { + ok = 0; + goto END; + } // take D=1 because most frequent, but not actually used + L -= LZFSE_ENCODE_MAX_L_VALUE; + } + + // Split if M too large + while (M > LZFSE_ENCODE_MAX_M_VALUE) { + if (lzfse_push_lmd(s, L, LZFSE_ENCODE_MAX_M_VALUE, D) != 0) { + ok = 0; + goto END; + } + L = 0; + M -= LZFSE_ENCODE_MAX_M_VALUE; + } + + // L,M in range + if (L > 0 || M > 0) { + if (lzfse_push_lmd(s, L, M, D) != 0) { + ok = 0; + goto END; + } + L = M = 0; + (void)L; + (void)M; // dead stores + } + +END: + if (!ok) { + // Revert state + s->n_matches = n_matches0; + s->n_literals = n_literals0; + s->src_literal = src_literals0; + + return LZFSE_STATUS_DST_FULL; // state tables full + } + + return LZFSE_STATUS_OK; // OK +} + +/*! @abstract Backend: add MATCH to state S. Encode block if necessary, when + * state is full. + * @return LZFSE_STATUS_OK if OK. + * @return LZFSE_STATUS_DST_FULL if the match can't be added, meaning one of the + * buffers is full. In that case the state is not modified. */ +static int lzfse_backend_match(lzfse_encoder_state *s, + const lzfse_match *match) { + // Try to push the match in state + if (lzfse_push_match(s, match) == LZFSE_STATUS_OK) + return LZFSE_STATUS_OK; // OK, match added to state + + // Here state tables are full, try to emit block + if (lzfse_encode_matches(s) != LZFSE_STATUS_OK) + return LZFSE_STATUS_DST_FULL; // DST full, match not added + + // Here block has been emitted, re-try to push the match in state + return lzfse_push_match(s, match); +} + +/*! @abstract Backend: add L literals to state S. Encode block if necessary, + * when state is full. + * @return LZFSE_STATUS_OK if OK. + * @return LZFSE_STATUS_DST_FULL if the literals can't be added, meaning one of + * the buffers is full. In that case the state is not modified. */ +static int lzfse_backend_literals(lzfse_encoder_state *s, lzfse_offset L) { + // Create a fake match with M=0, D=1 + lzfse_match match; + lzfse_offset pos = s->src_literal + L; + match.pos = pos; + match.ref = match.pos - 1; + match.length = 0; + return lzfse_backend_match(s, &match); +} + +/*! @abstract Backend: flush final block, and emit end of stream + * @return LZFSE_STATUS_OK if OK. + * @return LZFSE_STATUS_DST_FULL if either the final block, or the end-of-stream + * can't be added, meaning one of the buffers is full. If the block was emitted, + * the state is updated to reflect this. Otherwise, it is left unchanged. */ +static int lzfse_backend_end_of_stream(lzfse_encoder_state *s) { + // Final match triggers write, otherwise emit blocks when we have enough + // matches stored + if (lzfse_encode_matches(s) != LZFSE_STATUS_OK) + return LZFSE_STATUS_DST_FULL; // DST full + + // Emit end-of-stream block + if (s->dst + 4 > s->dst_end) + return LZFSE_STATUS_DST_FULL; // DST full + store4(s->dst, LZFSE_ENDOFSTREAM_BLOCK_MAGIC); + s->dst += 4; + + return LZFSE_STATUS_OK; // OK +} + +// =============================================================== +// Encoder state management + +/*! @abstract Initialize state: + * @code + * - hash table with all invalid pos, and value 0. + * - pending match to NO_MATCH. + * - src_literal to 0. + * - d_prev to 0. + @endcode + * @return LZFSE_STATUS_OK */ +int lzfse_encode_init(lzfse_encoder_state *s) { + const lzfse_match NO_MATCH = {0}; + lzfse_history_set line; + for (int i = 0; i < LZFSE_ENCODE_HASH_WIDTH; i++) { + line.pos[i] = -4 * LZFSE_ENCODE_MAX_D_VALUE; // invalid pos + line.value[i] = 0; + } + // Fill table + for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++) + s->history_table[i] = line; + s->pending = NO_MATCH; + s->src_literal = 0; + + return LZFSE_STATUS_OK; // OK +} + +/*! @abstract Translate state \p src forward by \p delta > 0. + * Offsets in \p src are updated backwards to point to the same positions. + * @return LZFSE_STATUS_OK */ +int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta) { + assert(delta >= 0); + if (delta == 0) + return LZFSE_STATUS_OK; // OK + + // SRC + s->src += delta; + + // Offsets in SRC + s->src_end -= delta; + s->src_encode_i -= delta; + s->src_encode_end -= delta; + s->src_literal -= delta; + + // Pending match + s->pending.pos -= delta; + s->pending.ref -= delta; + + // history_table positions, translated, and clamped to invalid pos + int32_t invalidPos = -4 * LZFSE_ENCODE_MAX_D_VALUE; + for (int i = 0; i < LZFSE_ENCODE_HASH_VALUES; i++) { + int32_t *p = &(s->history_table[i].pos[0]); + for (int j = 0; j < LZFSE_ENCODE_HASH_WIDTH; j++) { + lzfse_offset newPos = p[j] - delta; // translate + p[j] = (int32_t)((newPos < invalidPos) ? invalidPos : newPos); // clamp + } + } + + return LZFSE_STATUS_OK; // OK +} + +// =============================================================== +// Encoder front end + +int lzfse_encode_base(lzfse_encoder_state *s) { + lzfse_history_set *history_table = s->history_table; + lzfse_history_set *hashLine = 0; + lzfse_history_set newH; + const lzfse_match NO_MATCH = {0}; + int ok = 1; + + memset(&newH, 0x00, sizeof(newH)); + + // 8 byte padding at end of buffer + s->src_encode_end = s->src_end - 8; + for (; s->src_encode_i < s->src_encode_end; s->src_encode_i++) { + lzfse_offset pos = s->src_encode_i; // pos >= 0 + + // Load 4 byte value and get hash line + uint32_t x = load4(s->src + pos); + hashLine = history_table + hashX(x); + lzfse_history_set h = *hashLine; + + // Prepare next hash line (component 0 is the most recent) to prepare new + // entries (stored later) + { + newH.pos[0] = (int32_t)pos; + for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++) + newH.pos[k + 1] = h.pos[k]; + newH.value[0] = x; + for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH - 1; k++) + newH.value[k + 1] = h.value[k]; + } + + // Do not look for a match if we are still covered by a previous match + if (pos < s->src_literal) + goto END_POS; + + // Search best incoming match + lzfse_match incoming = {.pos = pos, .ref = 0, .length = 0}; + + // Check for matches. We consider matches of length >= 4 only. + for (int k = 0; k < LZFSE_ENCODE_HASH_WIDTH; k++) { + uint32_t d = h.value[k] ^ x; + if (d) + continue; // no 4 byte match + int32_t ref = h.pos[k]; + if (ref + LZFSE_ENCODE_MAX_D_VALUE < pos) + continue; // too far + + const uint8_t *src_ref = s->src + ref; + const uint8_t *src_pos = s->src + pos; + uint32_t length = 4; + uint32_t maxLength = + (uint32_t)(s->src_end - pos - 8); // ensure we don't hit the end of SRC + while (length < maxLength) { + uint64_t d = load8(src_ref + length) ^ load8(src_pos + length); + if (d == 0) { + length += 8; + continue; + } + + length += + (__builtin_ctzll(d) >> 3); // ctzll must be called only with D != 0 + break; + } + if (length > incoming.length) { + incoming.length = length; + incoming.ref = ref; + } // keep if longer + } + + // No incoming match? + if (incoming.length == 0) { + // We may still want to emit some literals here, to not lag too far behind + // the current search point, and avoid + // ending up with a literal block not fitting in the state. + lzfse_offset n_literals = pos - s->src_literal; + // The threshold here should be larger than a couple of MAX_L_VALUE, and + // much smaller than LITERALS_PER_BLOCK + if (n_literals > 8 * LZFSE_ENCODE_MAX_L_VALUE) { + // Here, we need to consume some literals. Emit pending match if there + // is one + if (s->pending.length > 0) { + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) { + ok = 0; + goto END; + } + s->pending = NO_MATCH; + } else { + // No pending match, emit a full LZFSE_ENCODE_MAX_L_VALUE block of + // literals + if (lzfse_backend_literals(s, LZFSE_ENCODE_MAX_L_VALUE) != + LZFSE_STATUS_OK) { + ok = 0; + goto END; + } + } + } + goto END_POS; // no incoming match + } + + // Limit match length (it may still be expanded backwards, but this is + // bounded by the limit on literals we tested before) + if (incoming.length > LZFSE_ENCODE_MAX_MATCH_LENGTH) { + incoming.length = LZFSE_ENCODE_MAX_MATCH_LENGTH; + } + + // Expand backwards (since this is expensive, we do this for the best match + // only) + while (incoming.pos > s->src_literal && incoming.ref > 0 && + s->src[incoming.ref - 1] == s->src[incoming.pos - 1]) { + incoming.pos--; + incoming.ref--; + } + incoming.length += pos - incoming.pos; // update length after expansion + + // Match filtering heuristic (from LZVN). INCOMING is always defined here. + + // Incoming is 'good', emit incoming + if (incoming.length >= LZFSE_ENCODE_GOOD_MATCH) { + if (lzfse_backend_match(s, &incoming) != LZFSE_STATUS_OK) { + ok = 0; + goto END; + } + s->pending = NO_MATCH; + goto END_POS; + } + + // No pending, keep incoming + if (s->pending.length == 0) { + s->pending = incoming; + goto END_POS; + } + + // No overlap, emit pending, keep incoming + if (s->pending.pos + s->pending.length <= incoming.pos) { + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) { + ok = 0; + goto END; + } + s->pending = incoming; + goto END_POS; + } + + // Overlap: emit longest + if (incoming.length > s->pending.length) { + if (lzfse_backend_match(s, &incoming) != LZFSE_STATUS_OK) { + ok = 0; + goto END; + } + } else { + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) { + ok = 0; + goto END; + } + } + s->pending = NO_MATCH; + + END_POS: + // We are done with this src_encode_i. + // Update state now (s->pending has already been updated). + *hashLine = newH; + } + +END: + return ok ? LZFSE_STATUS_OK : LZFSE_STATUS_DST_FULL; +} + +int lzfse_encode_finish(lzfse_encoder_state *s) { + const lzfse_match NO_MATCH = {0}; + + // Emit pending match + if (s->pending.length > 0) { + if (lzfse_backend_match(s, &s->pending) != LZFSE_STATUS_OK) + return LZFSE_STATUS_DST_FULL; + s->pending = NO_MATCH; + } + + // Emit final literals if any + lzfse_offset L = s->src_end - s->src_literal; + if (L > 0) { + if (lzfse_backend_literals(s, L) != LZFSE_STATUS_OK) + return LZFSE_STATUS_DST_FULL; + } + + // Emit all matches, and end-of-stream block + if (lzfse_backend_end_of_stream(s) != LZFSE_STATUS_OK) + return LZFSE_STATUS_DST_FULL; + + return LZFSE_STATUS_OK; +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_encode_tables.h b/view/kernelcache/core/transformers/liblzfse/lzfse_encode_tables.h new file mode 100644 index 00000000..1b78c523 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_encode_tables.h @@ -0,0 +1,218 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZFSE_ENCODE_TABLES_H +#define LZFSE_ENCODE_TABLES_H + +#if defined(_MSC_VER) && !defined(__clang__) +# define inline __inline +#endif + +static inline uint8_t l_base_from_value(int32_t value) { + static const uint8_t sym[LZFSE_ENCODE_MAX_L_VALUE + 1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, + 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19}; + return sym[value]; +} +static inline uint8_t m_base_from_value(int32_t value) { + static const uint8_t sym[LZFSE_ENCODE_MAX_M_VALUE + 1] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 16, + 16, 16, 16, 16, 16, 16, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, + 17, 17, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, 19, + 19, 19}; + return sym[value]; +} + +static inline uint8_t d_base_from_value(int32_t value) { + static const uint8_t sym[64 * 4] = { + 0, 1, 2, 3, 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8, 9, 9, + 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, + 13, 13, 13, 13, 13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, + 15, 15, 15, 15, 15, 15, 16, 16, 16, 16, 16, 17, 18, 19, 20, 20, 21, 21, + 22, 22, 23, 23, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26, 27, 27, + 27, 27, 28, 28, 28, 28, 28, 28, 28, 28, 29, 29, 29, 29, 29, 29, 29, 29, + 30, 30, 30, 30, 30, 30, 30, 30, 31, 31, 31, 31, 31, 31, 31, 31, 32, 32, + 32, 32, 32, 33, 34, 35, 36, 36, 37, 37, 38, 38, 39, 39, 40, 40, 40, 40, + 41, 41, 41, 41, 42, 42, 42, 42, 43, 43, 43, 43, 44, 44, 44, 44, 44, 44, + 44, 44, 45, 45, 45, 45, 45, 45, 45, 45, 46, 46, 46, 46, 46, 46, 46, 46, + 47, 47, 47, 47, 47, 47, 47, 47, 48, 48, 48, 48, 48, 49, 50, 51, 52, 52, + 53, 53, 54, 54, 55, 55, 56, 56, 56, 56, 57, 57, 57, 57, 58, 58, 58, 58, + 59, 59, 59, 59, 60, 60, 60, 60, 60, 60, 60, 60, 61, 61, 61, 61, 61, 61, + 61, 61, 62, 62, 62, 62, 62, 62, 62, 62, 63, 63, 63, 63, 63, 63, 63, 63, + 0, 0, 0, 0}; + int index = 0; + int in_range_k; + in_range_k = (value >= 0 && value < 60); + index |= (((value - 0) >> 0) + 0) & -in_range_k; + in_range_k = (value >= 60 && value < 1020); + index |= (((value - 60) >> 4) + 64) & -in_range_k; + in_range_k = (value >= 1020 && value < 16380); + index |= (((value - 1020) >> 8) + 128) & -in_range_k; + in_range_k = (value >= 16380 && value < 262140); + index |= (((value - 16380) >> 12) + 192) & -in_range_k; + return sym[index & 255]; +} + +#endif // LZFSE_ENCODE_TABLES_H diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_fse.c b/view/kernelcache/core/transformers/liblzfse/lzfse_fse.c new file mode 100644 index 00000000..631ada06 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_fse.c @@ -0,0 +1,216 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#include "lzfse_internal.h" + +// Initialize encoder table T[NSYMBOLS]. +// NSTATES = sum FREQ[i] is the number of states (a power of 2) +// NSYMBOLS is the number of symbols. +// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i] +// >= 0. +// Some symbols may have a 0 frequency. In that case, they should not be +// present in the data. +void fse_init_encoder_table(int nstates, int nsymbols, + const uint16_t *__restrict freq, + fse_encoder_entry *__restrict t) { + int offset = 0; // current offset + int n_clz = __builtin_clz(nstates); + for (int i = 0; i < nsymbols; i++) { + int f = (int)freq[i]; + if (f == 0) + continue; // skip this symbol, no occurrences + int k = + __builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N + t[i].s0 = (int16_t)((f << k) - nstates); + t[i].k = (int16_t)k; + t[i].delta0 = (int16_t)(offset - f + (nstates >> k)); + t[i].delta1 = (int16_t)(offset - f + (nstates >> (k - 1))); + offset += f; + } +} + +// Initialize decoder table T[NSTATES]. +// NSTATES = sum FREQ[i] is the number of states (a power of 2) +// NSYMBOLS is the number of symbols. +// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i] +// >= 0. +// Some symbols may have a 0 frequency. In that case, they should not be +// present in the data. +int fse_init_decoder_table(int nstates, int nsymbols, + const uint16_t *__restrict freq, + int32_t *__restrict t) { + assert(nsymbols <= 256); + assert(fse_check_freq(freq, nsymbols, nstates) == 0); + int n_clz = __builtin_clz(nstates); + int sum_of_freq = 0; + for (int i = 0; i < nsymbols; i++) { + int f = (int)freq[i]; + if (f == 0) + continue; // skip this symbol, no occurrences + + sum_of_freq += f; + + if (sum_of_freq > nstates) { + return -1; + } + + int k = + __builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N + int j0 = ((2 * nstates) >> k) - f; + + // Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F + for (int j = 0; j < f; j++) { + fse_decoder_entry e; + + e.symbol = (uint8_t)i; + if (j < j0) { + e.k = (int8_t)k; + e.delta = (int16_t)(((f + j) << k) - nstates); + } else { + e.k = (int8_t)(k - 1); + e.delta = (int16_t)((j - j0) << (k - 1)); + } + + memcpy(t, &e, sizeof(e)); + t++; + } + } + + return 0; // OK +} + +// Initialize value decoder table T[NSTATES]. +// NSTATES = sum FREQ[i] is the number of states (a power of 2) +// NSYMBOLS is the number of symbols. +// FREQ[NSYMBOLS] is a normalized histogram of symbol frequencies, with FREQ[i] +// >= 0. +// SYMBOL_VBITS[NSYMBOLS] and SYMBOLS_VBASE[NSYMBOLS] are the number of value +// bits to read and the base value for each symbol. +// Some symbols may have a 0 frequency. In that case, they should not be +// present in the data. +void fse_init_value_decoder_table(int nstates, int nsymbols, + const uint16_t *__restrict freq, + const uint8_t *__restrict symbol_vbits, + const int32_t *__restrict symbol_vbase, + fse_value_decoder_entry *__restrict t) { + assert(nsymbols <= 256); + assert(fse_check_freq(freq, nsymbols, nstates) == 0); + + int n_clz = __builtin_clz(nstates); + for (int i = 0; i < nsymbols; i++) { + int f = (int)freq[i]; + if (f == 0) + continue; // skip this symbol, no occurrences + + int k = + __builtin_clz(f) - n_clz; // shift needed to ensure N <= (F<<K) < 2*N + int j0 = ((2 * nstates) >> k) - f; + + fse_value_decoder_entry ei = {0}; + ei.value_bits = symbol_vbits[i]; + ei.vbase = symbol_vbase[i]; + + // Initialize all states S reached by this symbol: OFFSET <= S < OFFSET + F + for (int j = 0; j < f; j++) { + fse_value_decoder_entry e = ei; + + if (j < j0) { + e.total_bits = (uint8_t)k + e.value_bits; + e.delta = (int16_t)(((f + j) << k) - nstates); + } else { + e.total_bits = (uint8_t)(k - 1) + e.value_bits; + e.delta = (int16_t)((j - j0) << (k - 1)); + } + + memcpy(t, &e, 8); + t++; + } + } +} + +// Remove states from symbols until the correct number of states is used. +static void fse_adjust_freqs(uint16_t *freq, int overrun, int nsymbols) { + for (int shift = 3; overrun != 0; shift--) { + for (int sym = 0; sym < nsymbols; sym++) { + if (freq[sym] > 1) { + int n = (freq[sym] - 1) >> shift; + if (n > overrun) + n = overrun; + freq[sym] -= n; + overrun -= n; + if (overrun == 0) + break; + } + } + } +} + +// Normalize a table T[NSYMBOLS] of occurrences to FREQ[NSYMBOLS]. +void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t, + uint16_t *__restrict freq) { + uint32_t s_count = 0; + int remaining = nstates; // must be signed; this may become < 0 + int max_freq = 0; + int max_freq_sym = 0; + int shift = __builtin_clz(nstates) - 1; + uint32_t highprec_step; + + // Compute the total number of symbol occurrences + for (int i = 0; i < nsymbols; i++) + s_count += t[i]; + + if (s_count == 0) + highprec_step = 0; // no symbols used + else + highprec_step = ((uint32_t)1 << 31) / s_count; + + for (int i = 0; i < nsymbols; i++) { + + // Rescale the occurrence count to get the normalized frequency. + // Round up if the fractional part is >= 0.5; otherwise round down. + // For efficiency, we do this calculation using integer arithmetic. + int f = (((t[i] * highprec_step) >> shift) + 1) >> 1; + + // If a symbol was used, it must be given a nonzero normalized frequency. + if (f == 0 && t[i] != 0) + f = 1; + + freq[i] = f; + remaining -= f; + + // Remember the maximum frequency and which symbol had it. + if (f > max_freq) { + max_freq = f; + max_freq_sym = i; + } + } + + // If there remain states to be assigned, then just assign them to the most + // frequent symbol. Alternatively, if we assigned more states than were + // actually available, then either remove states from the most frequent symbol + // (for minor overruns) or use the slower adjustment algorithm (for major + // overruns). + if (-remaining < (max_freq >> 2)) { + freq[max_freq_sym] += remaining; + } else { + fse_adjust_freqs(freq, -remaining, nsymbols); + } +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_fse.h b/view/kernelcache/core/transformers/liblzfse/lzfse_fse.h new file mode 100644 index 00000000..88460129 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_fse.h @@ -0,0 +1,631 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// Finite state entropy coding (FSE) +// This is an implementation of the tANS algorithm described by Jarek Duda, +// we use the more descriptive name "Finite State Entropy". + +#pragma once + +#include <assert.h> +#include <stddef.h> +#include <stdint.h> +#include <stdlib.h> +#include <string.h> + +// Select between 32/64-bit I/O streams for FSE. Note that the FSE stream +// size need not match the word size of the machine, but in practice you +// want to use 64b streams on 64b systems for better performance. +#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__) +#define FSE_IOSTREAM_64 1 +#else +#define FSE_IOSTREAM_64 0 +#endif + +#if defined(_MSC_VER) && !defined(__clang__) +# define FSE_INLINE __forceinline +# define inline __inline +# pragma warning(disable : 4068) // warning C4068: unknown pragma +#else +# define FSE_INLINE static inline __attribute__((__always_inline__)) +#endif + +// MARK: - Bit utils + +/*! @abstract Signed type used to represent bit count. */ +typedef int32_t fse_bit_count; + +/*! @abstract Unsigned type used to represent FSE state. */ +typedef uint16_t fse_state; + +// Mask the NBITS lsb of X. 0 <= NBITS < 64 +static inline uint64_t fse_mask_lsb64(uint64_t x, fse_bit_count nbits) { + static const uint64_t mtable[65] = { + 0x0000000000000000LLU, 0x0000000000000001LLU, 0x0000000000000003LLU, + 0x0000000000000007LLU, 0x000000000000000fLLU, 0x000000000000001fLLU, + 0x000000000000003fLLU, 0x000000000000007fLLU, 0x00000000000000ffLLU, + 0x00000000000001ffLLU, 0x00000000000003ffLLU, 0x00000000000007ffLLU, + 0x0000000000000fffLLU, 0x0000000000001fffLLU, 0x0000000000003fffLLU, + 0x0000000000007fffLLU, 0x000000000000ffffLLU, 0x000000000001ffffLLU, + 0x000000000003ffffLLU, 0x000000000007ffffLLU, 0x00000000000fffffLLU, + 0x00000000001fffffLLU, 0x00000000003fffffLLU, 0x00000000007fffffLLU, + 0x0000000000ffffffLLU, 0x0000000001ffffffLLU, 0x0000000003ffffffLLU, + 0x0000000007ffffffLLU, 0x000000000fffffffLLU, 0x000000001fffffffLLU, + 0x000000003fffffffLLU, 0x000000007fffffffLLU, 0x00000000ffffffffLLU, + 0x00000001ffffffffLLU, 0x00000003ffffffffLLU, 0x00000007ffffffffLLU, + 0x0000000fffffffffLLU, 0x0000001fffffffffLLU, 0x0000003fffffffffLLU, + 0x0000007fffffffffLLU, 0x000000ffffffffffLLU, 0x000001ffffffffffLLU, + 0x000003ffffffffffLLU, 0x000007ffffffffffLLU, 0x00000fffffffffffLLU, + 0x00001fffffffffffLLU, 0x00003fffffffffffLLU, 0x00007fffffffffffLLU, + 0x0000ffffffffffffLLU, 0x0001ffffffffffffLLU, 0x0003ffffffffffffLLU, + 0x0007ffffffffffffLLU, 0x000fffffffffffffLLU, 0x001fffffffffffffLLU, + 0x003fffffffffffffLLU, 0x007fffffffffffffLLU, 0x00ffffffffffffffLLU, + 0x01ffffffffffffffLLU, 0x03ffffffffffffffLLU, 0x07ffffffffffffffLLU, + 0x0fffffffffffffffLLU, 0x1fffffffffffffffLLU, 0x3fffffffffffffffLLU, + 0x7fffffffffffffffLLU, 0xffffffffffffffffLLU, + }; + return x & mtable[nbits]; +} + +// Mask the NBITS lsb of X. 0 <= NBITS < 32 +static inline uint32_t fse_mask_lsb32(uint32_t x, fse_bit_count nbits) { + static const uint32_t mtable[33] = { + 0x0000000000000000U, 0x0000000000000001U, 0x0000000000000003U, + 0x0000000000000007U, 0x000000000000000fU, 0x000000000000001fU, + 0x000000000000003fU, 0x000000000000007fU, 0x00000000000000ffU, + 0x00000000000001ffU, 0x00000000000003ffU, 0x00000000000007ffU, + 0x0000000000000fffU, 0x0000000000001fffU, 0x0000000000003fffU, + 0x0000000000007fffU, 0x000000000000ffffU, 0x000000000001ffffU, + 0x000000000003ffffU, 0x000000000007ffffU, 0x00000000000fffffU, + 0x00000000001fffffU, 0x00000000003fffffU, 0x00000000007fffffU, + 0x0000000000ffffffU, 0x0000000001ffffffU, 0x0000000003ffffffU, + 0x0000000007ffffffU, 0x000000000fffffffU, 0x000000001fffffffU, + 0x000000003fffffffU, 0x000000007fffffffU, 0x00000000ffffffffU, + }; + return x & mtable[nbits]; +} + +/*! @abstract Select \c nbits at index \c start from \c x. + * 0 <= start <= start+nbits <= 64 */ +FSE_INLINE uint64_t fse_extract_bits64(uint64_t x, fse_bit_count start, + fse_bit_count nbits) { +#if defined(__GNUC__) + // If START and NBITS are constants, map to bit-field extraction instructions + if (__builtin_constant_p(start) && __builtin_constant_p(nbits)) + return (x >> start) & ((1LLU << nbits) - 1LLU); +#endif + + // Otherwise, shift and mask + return fse_mask_lsb64(x >> start, nbits); +} + +/*! @abstract Select \c nbits at index \c start from \c x. + * 0 <= start <= start+nbits <= 32 */ +FSE_INLINE uint32_t fse_extract_bits32(uint32_t x, fse_bit_count start, + fse_bit_count nbits) { +#if defined(__GNUC__) + // If START and NBITS are constants, map to bit-field extraction instructions + if (__builtin_constant_p(start) && __builtin_constant_p(nbits)) + return (x >> start) & ((1U << nbits) - 1U); +#endif + + // Otherwise, shift and mask + return fse_mask_lsb32(x >> start, nbits); +} + +// MARK: - Bit stream + +// I/O streams +// The streams can be shared between several FSE encoders/decoders, which is why +// they are not in the state struct + +/*! @abstract Output stream, 64-bit accum. */ +typedef struct { + uint64_t accum; // Output bits + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0 +} fse_out_stream64; + +/*! @abstract Output stream, 32-bit accum. */ +typedef struct { + uint32_t accum; // Output bits + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0 +} fse_out_stream32; + +/*! @abstract Object representing an input stream. */ +typedef struct { + uint64_t accum; // Input bits + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0 +} fse_in_stream64; + +/*! @abstract Object representing an input stream. */ +typedef struct { + uint32_t accum; // Input bits + fse_bit_count accum_nbits; // Number of valid bits in ACCUM, other bits are 0 +} fse_in_stream32; + +/*! @abstract Initialize an output stream object. */ +FSE_INLINE void fse_out_init64(fse_out_stream64 *s) { + s->accum = 0; + s->accum_nbits = 0; +} + +/*! @abstract Initialize an output stream object. */ +FSE_INLINE void fse_out_init32(fse_out_stream32 *s) { + s->accum = 0; + s->accum_nbits = 0; +} + +/*! @abstract Write full bytes from the accumulator to output buffer, ensuring + * accum_nbits is in [0, 7]. + * We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all + * cases. + * @note *pbuf is incremented by the number of written bytes. */ +FSE_INLINE void fse_out_flush64(fse_out_stream64 *s, uint8_t **pbuf) { + fse_bit_count nbits = + s->accum_nbits & -8; // number of bits written, multiple of 8 + + // Write 8 bytes of current accumulator + memcpy(*pbuf, &(s->accum), 8); + *pbuf += (nbits >> 3); // bytes + + // Update state + s->accum >>= nbits; // remove nbits + s->accum_nbits -= nbits; + + assert(s->accum_nbits >= 0 && s->accum_nbits <= 7); + assert(s->accum_nbits == 64 || (s->accum >> s->accum_nbits) == 0); +} + +/*! @abstract Write full bytes from the accumulator to output buffer, ensuring + * accum_nbits is in [0, 7]. + * We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all + * cases. + * @note *pbuf is incremented by the number of written bytes. */ +FSE_INLINE void fse_out_flush32(fse_out_stream32 *s, uint8_t **pbuf) { + fse_bit_count nbits = + s->accum_nbits & -8; // number of bits written, multiple of 8 + + // Write 4 bytes of current accumulator + memcpy(*pbuf, &(s->accum), 4); + *pbuf += (nbits >> 3); // bytes + + // Update state + s->accum >>= nbits; // remove nbits + s->accum_nbits -= nbits; + + assert(s->accum_nbits >= 0 && s->accum_nbits <= 7); + assert(s->accum_nbits == 32 || (s->accum >> s->accum_nbits) == 0); +} + +/*! @abstract Write the last bytes from the accumulator to output buffer, + * ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed. + * We assume we can write 8 bytes to the output buffer \c (*pbuf[0..7]) in all + * cases. + * @note *pbuf is incremented by the number of written bytes. */ +FSE_INLINE void fse_out_finish64(fse_out_stream64 *s, uint8_t **pbuf) { + fse_bit_count nbits = + (s->accum_nbits + 7) & -8; // number of bits written, multiple of 8 + + // Write 8 bytes of current accumulator + memcpy(*pbuf, &(s->accum), 8); + *pbuf += (nbits >> 3); // bytes + + // Update state + s->accum = 0; // remove nbits + s->accum_nbits -= nbits; + + assert(s->accum_nbits >= -7 && s->accum_nbits <= 0); +} + +/*! @abstract Write the last bytes from the accumulator to output buffer, + * ensuring accum_nbits is in [-7, 0]. Bits are padded with 0 if needed. + * We assume we can write 4 bytes to the output buffer \c (*pbuf[0..3]) in all + * cases. + * @note *pbuf is incremented by the number of written bytes. */ +FSE_INLINE void fse_out_finish32(fse_out_stream32 *s, uint8_t **pbuf) { + fse_bit_count nbits = + (s->accum_nbits + 7) & -8; // number of bits written, multiple of 8 + + // Write 8 bytes of current accumulator + memcpy(*pbuf, &(s->accum), 4); + *pbuf += (nbits >> 3); // bytes + + // Update state + s->accum = 0; // remove nbits + s->accum_nbits -= nbits; + + assert(s->accum_nbits >= -7 && s->accum_nbits <= 0); +} + +/*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have: + * 0 <= b < 2^n, and N + s->accum_nbits <= 64. + * @note The caller must ensure out_flush is called \b before the accumulator + * overflows to more than 64 bits. */ +FSE_INLINE void fse_out_push64(fse_out_stream64 *s, fse_bit_count n, + uint64_t b) { + s->accum |= b << s->accum_nbits; + s->accum_nbits += n; + + assert(s->accum_nbits >= 0 && s->accum_nbits <= 64); + assert(s->accum_nbits == 64 || (s->accum >> s->accum_nbits) == 0); +} + +/*! @abstract Accumulate \c n bits \c b to output stream \c s. We \b must have: + * 0 <= n < 2^n, and n + s->accum_nbits <= 32. + * @note The caller must ensure out_flush is called \b before the accumulator + * overflows to more than 32 bits. */ +FSE_INLINE void fse_out_push32(fse_out_stream32 *s, fse_bit_count n, + uint32_t b) { + s->accum |= b << s->accum_nbits; + s->accum_nbits += n; + + assert(s->accum_nbits >= 0 && s->accum_nbits <= 32); + assert(s->accum_nbits == 32 || (s->accum >> s->accum_nbits) == 0); +} + +#if FSE_IOSTREAM_64 +#define DEBUG_CHECK_INPUT_STREAM_PARAMETERS \ + assert(s->accum_nbits >= 56 && s->accum_nbits < 64); \ + assert((s->accum >> s->accum_nbits) == 0); +#else +#define DEBUG_CHECK_INPUT_STREAM_PARAMETERS \ + assert(s->accum_nbits >= 24 && s->accum_nbits < 32); \ + assert((s->accum >> s->accum_nbits) == 0); +#endif + +/*! @abstract Initialize the fse input stream so that accum holds between 56 + * and 63 bits. We never want to have 64 bits in the stream, because that allows + * us to avoid a special case in the fse_in_pull function (eliminating an + * unpredictable branch), while not requiring any additional fse_flush + * operations. This is why we have the special case for n == 0 (in which case + * we want to load only 7 bytes instead of 8). */ +FSE_INLINE int fse_in_checked_init64(fse_in_stream64 *s, fse_bit_count n, + const uint8_t **pbuf, + const uint8_t *buf_start) { + if (n) { + if (*pbuf < buf_start + 8) + return -1; // out of range + *pbuf -= 8; + memcpy(&(s->accum), *pbuf, 8); + s->accum_nbits = n + 64; + } else { + if (*pbuf < buf_start + 7) + return -1; // out of range + *pbuf -= 7; + memcpy(&(s->accum), *pbuf, 7); + s->accum &= 0xffffffffffffff; + s->accum_nbits = n + 56; + } + + if ((s->accum_nbits < 56 || s->accum_nbits >= 64) || + ((s->accum >> s->accum_nbits) != 0)) { + return -1; // the incoming input is wrong (encoder should have zeroed the + // upper bits) + } + + return 0; // OK +} + +/*! @abstract Identical to previous function, but for 32-bit operation + * (resulting bit count is between 24 and 31 bits). */ +FSE_INLINE int fse_in_checked_init32(fse_in_stream32 *s, fse_bit_count n, + const uint8_t **pbuf, + const uint8_t *buf_start) { + if (n) { + if (*pbuf < buf_start + 4) + return -1; // out of range + *pbuf -= 4; + memcpy(&(s->accum), *pbuf, 4); + s->accum_nbits = n + 32; + } else { + if (*pbuf < buf_start + 3) + return -1; // out of range + *pbuf -= 3; + memcpy(&(s->accum), *pbuf, 3); + s->accum &= 0xffffff; + s->accum_nbits = n + 24; + } + + if ((s->accum_nbits < 24 || s->accum_nbits >= 32) || + ((s->accum >> s->accum_nbits) != 0)) { + return -1; // the incoming input is wrong (encoder should have zeroed the + // upper bits) + } + + return 0; // OK +} + +/*! @abstract Read in new bytes from buffer to ensure that we have a full + * complement of bits in the stream object (again, between 56 and 63 bits). + * checking the new value of \c *pbuf remains >= \c buf_start. + * @return 0 if OK. + * @return -1 on failure. */ +FSE_INLINE int fse_in_checked_flush64(fse_in_stream64 *s, const uint8_t **pbuf, + const uint8_t *buf_start) { + // Get number of bits to add to bring us into the desired range. + fse_bit_count nbits = (63 - s->accum_nbits) & -8; + // Convert bits to bytes and decrement buffer address, then load new data. + const uint8_t *buf = (*pbuf) - (nbits >> 3); + if (buf < buf_start) { + return -1; // out of range + } + *pbuf = buf; + uint64_t incoming; + memcpy(&incoming, buf, 8); + // Update the state object and verify its validity (in DEBUG). + s->accum = (s->accum << nbits) | fse_mask_lsb64(incoming, nbits); + s->accum_nbits += nbits; + DEBUG_CHECK_INPUT_STREAM_PARAMETERS + return 0; // OK +} + +/*! @abstract Identical to previous function (but again, we're only filling + * a 32-bit field with between 24 and 31 bits). */ +FSE_INLINE int fse_in_checked_flush32(fse_in_stream32 *s, const uint8_t **pbuf, + const uint8_t *buf_start) { + // Get number of bits to add to bring us into the desired range. + fse_bit_count nbits = (31 - s->accum_nbits) & -8; + + if (nbits > 0) { + // Convert bits to bytes and decrement buffer address, then load new data. + const uint8_t *buf = (*pbuf) - (nbits >> 3); + if (buf < buf_start) { + return -1; // out of range + } + + *pbuf = buf; + + uint32_t incoming = *((uint32_t *)buf); + + // Update the state object and verify its validity (in DEBUG). + s->accum = (s->accum << nbits) | fse_mask_lsb32(incoming, nbits); + s->accum_nbits += nbits; + } + DEBUG_CHECK_INPUT_STREAM_PARAMETERS + return 0; // OK +} + +/*! @abstract Pull n bits out of the fse stream object. */ +FSE_INLINE uint64_t fse_in_pull64(fse_in_stream64 *s, fse_bit_count n) { + assert(n >= 0 && n <= s->accum_nbits); + s->accum_nbits -= n; + uint64_t result = s->accum >> s->accum_nbits; + s->accum = fse_mask_lsb64(s->accum, s->accum_nbits); + return result; +} + +/*! @abstract Pull n bits out of the fse stream object. */ +FSE_INLINE uint32_t fse_in_pull32(fse_in_stream32 *s, fse_bit_count n) { + assert(n >= 0 && n <= s->accum_nbits); + s->accum_nbits -= n; + uint32_t result = s->accum >> s->accum_nbits; + s->accum = fse_mask_lsb32(s->accum, s->accum_nbits); + return result; +} + +// MARK: - Encode/Decode + +// Map to 32/64-bit implementations and types for I/O +#if FSE_IOSTREAM_64 + +typedef uint64_t fse_bits; +typedef fse_out_stream64 fse_out_stream; +typedef fse_in_stream64 fse_in_stream; +#define fse_mask_lsb fse_mask_lsb64 +#define fse_extract_bits fse_extract_bits64 +#define fse_out_init fse_out_init64 +#define fse_out_flush fse_out_flush64 +#define fse_out_finish fse_out_finish64 +#define fse_out_push fse_out_push64 +#define fse_in_init fse_in_checked_init64 +#define fse_in_checked_init fse_in_checked_init64 +#define fse_in_flush fse_in_checked_flush64 +#define fse_in_checked_flush fse_in_checked_flush64 +#define fse_in_flush2(_unused, _parameters, _unused2) 0 /* nothing */ +#define fse_in_checked_flush2(_unused, _parameters) /* nothing */ +#define fse_in_pull fse_in_pull64 + +#else + +typedef uint32_t fse_bits; +typedef fse_out_stream32 fse_out_stream; +typedef fse_in_stream32 fse_in_stream; +#define fse_mask_lsb fse_mask_lsb32 +#define fse_extract_bits fse_extract_bits32 +#define fse_out_init fse_out_init32 +#define fse_out_flush fse_out_flush32 +#define fse_out_finish fse_out_finish32 +#define fse_out_push fse_out_push32 +#define fse_in_init fse_in_checked_init32 +#define fse_in_checked_init fse_in_checked_init32 +#define fse_in_flush fse_in_checked_flush32 +#define fse_in_checked_flush fse_in_checked_flush32 +#define fse_in_flush2 fse_in_checked_flush32 +#define fse_in_checked_flush2 fse_in_checked_flush32 +#define fse_in_pull fse_in_pull32 + +#endif + +/*! @abstract Entry for one symbol in the encoder table (64b). */ +typedef struct { + int16_t s0; // First state requiring a K-bit shift + int16_t k; // States S >= S0 are shifted K bits. States S < S0 are + // shifted K-1 bits + int16_t delta0; // Relative increment used to compute next state if S >= S0 + int16_t delta1; // Relative increment used to compute next state if S < S0 +} fse_encoder_entry; + +/*! @abstract Entry for one state in the decoder table (32b). */ +typedef struct { // DO NOT REORDER THE FIELDS + int8_t k; // Number of bits to read + uint8_t symbol; // Emitted symbol + int16_t delta; // Signed increment used to compute next state (+bias) +} fse_decoder_entry; + +/*! @abstract Entry for one state in the value decoder table (64b). */ +typedef struct { // DO NOT REORDER THE FIELDS + uint8_t total_bits; // state bits + extra value bits = shift for next decode + uint8_t value_bits; // extra value bits + int16_t delta; // state base (delta) + int32_t vbase; // value base +} fse_value_decoder_entry; + +/*! @abstract Encode SYMBOL using the encoder table, and update \c *pstate, + * \c out. + * @note The caller must ensure we have enough bits available in the output + * stream accumulator. */ +FSE_INLINE void fse_encode(fse_state *__restrict pstate, + const fse_encoder_entry *__restrict encoder_table, + fse_out_stream *__restrict out, uint8_t symbol) { + int s = *pstate; + fse_encoder_entry e = encoder_table[symbol]; + int s0 = e.s0; + int k = e.k; + int delta0 = e.delta0; + int delta1 = e.delta1; + + // Number of bits to write + int hi = s >= s0; + fse_bit_count nbits = hi ? k : (k - 1); + fse_state delta = hi ? delta0 : delta1; + + // Write lower NBITS of state + fse_bits b = fse_mask_lsb(s, nbits); + fse_out_push(out, nbits, b); + + // Update state with remaining bits and delta + *pstate = delta + (s >> nbits); +} + +/*! @abstract Decode and return symbol using the decoder table, and update + * \c *pstate, \c in. + * @note The caller must ensure we have enough bits available in the input + * stream accumulator. */ +FSE_INLINE uint8_t fse_decode(fse_state *__restrict pstate, + const int32_t *__restrict decoder_table, + fse_in_stream *__restrict in) { + int32_t e = decoder_table[*pstate]; + + // Update state from K bits of input + DELTA + *pstate = (fse_state)(e >> 16) + (fse_state)fse_in_pull(in, e & 0xff); + + // Return the symbol for this state + return fse_extract_bits(e, 8, 8); // symbol +} + +/*! @abstract Decode and return value using the decoder table, and update \c + * *pstate, \c in. + * \c value_decoder_table[nstates] + * @note The caller must ensure we have enough bits available in the input + * stream accumulator. */ +FSE_INLINE int32_t +fse_value_decode(fse_state *__restrict pstate, + const fse_value_decoder_entry *value_decoder_table, + fse_in_stream *__restrict in) { + fse_value_decoder_entry entry = value_decoder_table[*pstate]; + uint32_t state_and_value_bits = (uint32_t)fse_in_pull(in, entry.total_bits); + *pstate = + (fse_state)(entry.delta + (state_and_value_bits >> entry.value_bits)); + return (int32_t)(entry.vbase + + fse_mask_lsb(state_and_value_bits, entry.value_bits)); +} + +// MARK: - Tables + +// IMPORTANT: To properly decode an FSE encoded stream, both encoder/decoder +// tables shall be initialized with the same parameters, including the +// FREQ[NSYMBOL] array. +// + +/*! @abstract Sanity check on frequency table, verify sum of \c freq + * is <= \c number_of_states. */ +FSE_INLINE int fse_check_freq(const uint16_t *freq_table, + const size_t table_size, + const size_t number_of_states) { + size_t sum_of_freq = 0; + for (int i = 0; i < table_size; i++) { + sum_of_freq += freq_table[i]; + } + return (sum_of_freq > number_of_states) ? -1 : 0; +} + +/*! @abstract Initialize encoder table \c t[nsymbols]. + * + * @param nstates + * sum \c freq[i]; the number of states (a power of 2). + * + * @param nsymbols + * the number of symbols. + * + * @param freq[nsymbols] + * is a normalized histogram of symbol frequencies, with \c freq[i] >= 0. + * Some symbols may have a 0 frequency. In that case they should not be + * present in the data. + */ +void fse_init_encoder_table(int nstates, int nsymbols, + const uint16_t *__restrict freq, + fse_encoder_entry *__restrict t); + +/*! @abstract Initialize decoder table \c t[nstates]. + * + * @param nstates + * sum \c freq[i]; the number of states (a power of 2). + * + * @param nsymbols + * the number of symbols. + * + * @param feq[nsymbols] + * a normalized histogram of symbol frequencies, with \c freq[i] >= 0. + * Some symbols may have a 0 frequency. In that case they should not be + * present in the data. + * + * @return 0 if OK. + * @return -1 on failure. + */ +int fse_init_decoder_table(int nstates, int nsymbols, + const uint16_t *__restrict freq, + int32_t *__restrict t); + +/*! @abstract Initialize value decoder table \c t[nstates]. + * + * @param nstates + * sum \cfreq[i]; the number of states (a power of 2). + * + * @param nsymbols + * the number of symbols. + * + * @param freq[nsymbols] + * a normalized histogram of symbol frequencies, with \c freq[i] >= 0. + * \c symbol_vbits[nsymbols] and \c symbol_vbase[nsymbols] are the number of + * value bits to read and the base value for each symbol. + * Some symbols may have a 0 frequency. In that case they should not be + * present in the data. + */ +void fse_init_value_decoder_table(int nstates, int nsymbols, + const uint16_t *__restrict freq, + const uint8_t *__restrict symbol_vbits, + const int32_t *__restrict symbol_vbase, + fse_value_decoder_entry *__restrict t); + +/*! @abstract Normalize a table \c t[nsymbols] of occurrences to + * \c freq[nsymbols]. */ +void fse_normalize_freq(int nstates, int nsymbols, const uint32_t *__restrict t, + uint16_t *__restrict freq); diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_internal.h b/view/kernelcache/core/transformers/liblzfse/lzfse_internal.h new file mode 100644 index 00000000..7ea5a30a --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_internal.h @@ -0,0 +1,616 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZFSE_INTERNAL_H +#define LZFSE_INTERNAL_H + +// Unlike the tunable parameters defined in lzfse_tunables.h, you probably +// should not modify the values defined in this header. Doing so will either +// break the compressor, or result in a compressed data format that is +// incompatible. + +#include "lzfse_fse.h" +#include "lzfse_tunables.h" +#include <limits.h> +#include <stddef.h> +#include <stdint.h> + +#if defined(_MSC_VER) && !defined(__clang__) +# define LZFSE_INLINE __forceinline +# define __builtin_expect(X, Y) (X) +# define __attribute__(X) +# pragma warning(disable : 4068) // warning C4068: unknown pragma +#else +# define LZFSE_INLINE static inline __attribute__((__always_inline__)) +#endif + +// Implement GCC bit scan builtins for MSVC +#if defined(_MSC_VER) && !defined(__clang__) +#include <intrin.h> + +LZFSE_INLINE int __builtin_clz(unsigned int val) { + unsigned long r = 0; + if (_BitScanReverse(&r, val)) { + return 31 - r; + } + return 32; +} + +LZFSE_INLINE int __builtin_ctzl(unsigned long val) { + unsigned long r = 0; + if (_BitScanForward(&r, val)) { + return r; + } + return 32; +} + +LZFSE_INLINE int __builtin_ctzll(uint64_t val) { + unsigned long r = 0; +#if defined(_M_AMD64) || defined(_M_ARM) + if (_BitScanForward64(&r, val)) { + return r; + } +#else + if (_BitScanForward(&r, (uint32_t)val)) { + return r; + } + if (_BitScanForward(&r, (uint32_t)(val >> 32))) { + return 32 + r; + } +#endif + return 64; +} +#endif + +// Throughout LZFSE we refer to "L", "M" and "D"; these will always appear as +// a triplet, and represent a "usual" LZ-style literal and match pair. "L" +// is the number of literal bytes, "M" is the number of match bytes, and "D" +// is the match "distance"; the distance in bytes between the current pointer +// and the start of the match. +#define LZFSE_ENCODE_HASH_VALUES (1 << LZFSE_ENCODE_HASH_BITS) +#define LZFSE_ENCODE_L_SYMBOLS 20 +#define LZFSE_ENCODE_M_SYMBOLS 20 +#define LZFSE_ENCODE_D_SYMBOLS 64 +#define LZFSE_ENCODE_LITERAL_SYMBOLS 256 +#define LZFSE_ENCODE_L_STATES 64 +#define LZFSE_ENCODE_M_STATES 64 +#define LZFSE_ENCODE_D_STATES 256 +#define LZFSE_ENCODE_LITERAL_STATES 1024 +#define LZFSE_MATCHES_PER_BLOCK 10000 +#define LZFSE_LITERALS_PER_BLOCK (4 * LZFSE_MATCHES_PER_BLOCK) +#define LZFSE_DECODE_LITERALS_PER_BLOCK (4 * LZFSE_DECODE_MATCHES_PER_BLOCK) + +// LZFSE internal status. These values are used by internal LZFSE routines +// as return codes. There should not be any good reason to change their +// values; it is plausible that additional codes might be added in the +// future. +#define LZFSE_STATUS_OK 0 +#define LZFSE_STATUS_SRC_EMPTY -1 +#define LZFSE_STATUS_DST_FULL -2 +#define LZFSE_STATUS_ERROR -3 + +// Type representing an offset between elements in a buffer. On 64-bit +// systems, this is stored in a 64-bit container to avoid extra sign- +// extension operations in addressing arithmetic, but the value is always +// representable as a 32-bit signed value in LZFSE's usage. +#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__) +typedef int64_t lzfse_offset; +#else +typedef int32_t lzfse_offset; +#endif + +/*! @abstract History table set. Each line of the history table represents a set + * of candidate match locations, each of which begins with four bytes with the + * same hash. The table contains not only the positions, but also the first + * four bytes at each position. This doubles the memory footprint of the + * table, but allows us to quickly eliminate false-positive matches without + * doing any pointer chasing and without pulling in any additional cachelines. + * This provides a large performance win in practice. */ +typedef struct { + int32_t pos[LZFSE_ENCODE_HASH_WIDTH]; + uint32_t value[LZFSE_ENCODE_HASH_WIDTH]; +} lzfse_history_set; + +/*! @abstract An lzfse match is a sequence of bytes in the source buffer that + * exactly matches an earlier (but possibly overlapping) sequence of bytes in + * the same buffer. + * @code + * exeMPLARYexaMPLE + * | | | ||-|--- lzfse_match2.length=3 + * | | | ||----- lzfse_match2.pos + * | | |-|------ lzfse_match1.length=3 + * | | |-------- lzfse_match1.pos + * | |-------------- lzfse_match2.ref + * |----------------- lzfse_match1.ref + * @endcode + */ +typedef struct { + // Offset of the first byte in the match. + lzfse_offset pos; + // First byte of the source -- the earlier location in the buffer with the + // same contents. + lzfse_offset ref; + // Length of the match. + uint32_t length; +} lzfse_match; + +// MARK: - Encoder and Decoder state objects + +/*! @abstract Encoder state object. */ +typedef struct { + // Pointer to first byte of the source buffer. + const uint8_t *src; + // Length of the source buffer in bytes. Note that this is not a size_t, + // but rather lzfse_offset, which is a signed type. The largest + // representable buffer is 2GB, but arbitrarily large buffers may be + // handled by repeatedly calling the encoder function and "translating" + // the state between calls. When doing this, it is beneficial to use + // blocks smaller than 2GB in order to maintain residency in the last-level + // cache. Consult the implementation of lzfse_encode_buffer for details. + lzfse_offset src_end; + // Offset of the first byte of the next literal to encode in the source + // buffer. + lzfse_offset src_literal; + // Offset of the byte currently being checked for a match. + lzfse_offset src_encode_i; + // The last byte offset to consider for a match. In some uses it makes + // sense to use a smaller offset than src_end. + lzfse_offset src_encode_end; + // Pointer to the next byte to be written in the destination buffer. + uint8_t *dst; + // Pointer to the first byte of the destination buffer. + uint8_t *dst_begin; + // Pointer to one byte past the end of the destination buffer. + uint8_t *dst_end; + // Pending match; will be emitted unless a better match is found. + lzfse_match pending; + // The number of matches written so far. Note that there is no problem in + // using a 32-bit field for this quantity, because the state already limits + // us to at most 2GB of data; there cannot possibly be more matches than + // there are bytes in the input. + uint32_t n_matches; + // The number of literals written so far. + uint32_t n_literals; + // Lengths of found literals. + uint32_t l_values[LZFSE_MATCHES_PER_BLOCK]; + // Lengths of found matches. + uint32_t m_values[LZFSE_MATCHES_PER_BLOCK]; + // Distances of found matches. + uint32_t d_values[LZFSE_MATCHES_PER_BLOCK]; + // Concatenated literal bytes. + uint8_t literals[LZFSE_LITERALS_PER_BLOCK]; + // History table used to search for matches. Each entry of the table + // corresponds to a group of four byte sequences in the input stream + // that hash to the same value. + lzfse_history_set history_table[LZFSE_ENCODE_HASH_VALUES]; +} lzfse_encoder_state; + +/*! @abstract Decoder state object for lzfse compressed blocks. */ +typedef struct { + // Number of matches remaining in the block. + uint32_t n_matches; + // Number of bytes used to encode L, M, D triplets for the block. + uint32_t n_lmd_payload_bytes; + // Pointer to the next literal to emit. + const uint8_t *current_literal; + // L, M, D triplet for the match currently being emitted. This is used only + // if we need to restart after reaching the end of the destination buffer in + // the middle of a literal or match. + int32_t l_value, m_value, d_value; + // FSE stream object. + fse_in_stream lmd_in_stream; + // Offset of L,M,D encoding in the input buffer. Because we read through an + // FSE stream *backwards* while decoding, this is decremented as we move + // through a block. + uint32_t lmd_in_buf; + // The current state of the L, M, and D FSE decoders. + uint16_t l_state, m_state, d_state; + // Internal FSE decoder tables for the current block. These have + // alignment forced to 8 bytes to guarantee that a single state's + // entry cannot span two cachelines. + fse_value_decoder_entry l_decoder[LZFSE_ENCODE_L_STATES] __attribute__((__aligned__(8))); + fse_value_decoder_entry m_decoder[LZFSE_ENCODE_M_STATES] __attribute__((__aligned__(8))); + fse_value_decoder_entry d_decoder[LZFSE_ENCODE_D_STATES] __attribute__((__aligned__(8))); + int32_t literal_decoder[LZFSE_ENCODE_LITERAL_STATES]; + // The literal stream for the block, plus padding to allow for faster copy + // operations. + uint8_t literals[LZFSE_LITERALS_PER_BLOCK + 64]; +} lzfse_compressed_block_decoder_state; + +// Decoder state object for uncompressed blocks. +typedef struct { uint32_t n_raw_bytes; } uncompressed_block_decoder_state; + +/*! @abstract Decoder state object for lzvn-compressed blocks. */ +typedef struct { + uint32_t n_raw_bytes; + uint32_t n_payload_bytes; + uint32_t d_prev; +} lzvn_compressed_block_decoder_state; + +/*! @abstract Decoder state object. */ +typedef struct { + // Pointer to next byte to read from source buffer (this is advanced as we + // decode; src_begin describe the buffer and do not change). + const uint8_t *src; + // Pointer to first byte of source buffer. + const uint8_t *src_begin; + // Pointer to one byte past the end of the source buffer. + const uint8_t *src_end; + // Pointer to the next byte to write to destination buffer (this is advanced + // as we decode; dst_begin and dst_end describe the buffer and do not change). + uint8_t *dst; + // Pointer to first byte of destination buffer. + uint8_t *dst_begin; + // Pointer to one byte past the end of the destination buffer. + uint8_t *dst_end; + // 1 if we have reached the end of the stream, 0 otherwise. + int end_of_stream; + // magic number of the current block if we are within a block, + // LZFSE_NO_BLOCK_MAGIC otherwise. + uint32_t block_magic; + lzfse_compressed_block_decoder_state compressed_lzfse_block_state; + lzvn_compressed_block_decoder_state compressed_lzvn_block_state; + uncompressed_block_decoder_state uncompressed_block_state; +} lzfse_decoder_state; + +// MARK: - Block header objects + +#define LZFSE_NO_BLOCK_MAGIC 0x00000000 // 0 (invalid) +#define LZFSE_ENDOFSTREAM_BLOCK_MAGIC 0x24787662 // bvx$ (end of stream) +#define LZFSE_UNCOMPRESSED_BLOCK_MAGIC 0x2d787662 // bvx- (raw data) +#define LZFSE_COMPRESSEDV1_BLOCK_MAGIC 0x31787662 // bvx1 (lzfse compressed, uncompressed tables) +#define LZFSE_COMPRESSEDV2_BLOCK_MAGIC 0x32787662 // bvx2 (lzfse compressed, compressed tables) +#define LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC 0x6e787662 // bvxn (lzvn compressed) + +/*! @abstract Uncompressed block header in encoder stream. */ +typedef struct { + // Magic number, always LZFSE_UNCOMPRESSED_BLOCK_MAGIC. + uint32_t magic; + // Number of raw bytes in block. + uint32_t n_raw_bytes; +} uncompressed_block_header; + +/*! @abstract Compressed block header with uncompressed tables. */ +typedef struct { + // Magic number, always LZFSE_COMPRESSEDV1_BLOCK_MAGIC. + uint32_t magic; + // Number of decoded (output) bytes in block. + uint32_t n_raw_bytes; + // Number of encoded (source) bytes in block. + uint32_t n_payload_bytes; + // Number of literal bytes output by block (*not* the number of literals). + uint32_t n_literals; + // Number of matches in block (which is also the number of literals). + uint32_t n_matches; + // Number of bytes used to encode literals. + uint32_t n_literal_payload_bytes; + // Number of bytes used to encode matches. + uint32_t n_lmd_payload_bytes; + + // Final encoder states for the block, which will be the initial states for + // the decoder: + // Final accum_nbits for literals stream. + int32_t literal_bits; + // There are four interleaved streams of literals, so there are four final + // states. + uint16_t literal_state[4]; + // accum_nbits for the l, m, d stream. + int32_t lmd_bits; + // Final L (literal length) state. + uint16_t l_state; + // Final M (match length) state. + uint16_t m_state; + // Final D (match distance) state. + uint16_t d_state; + + // Normalized frequency tables for each stream. Sum of values in each + // array is the number of states. + uint16_t l_freq[LZFSE_ENCODE_L_SYMBOLS]; + uint16_t m_freq[LZFSE_ENCODE_M_SYMBOLS]; + uint16_t d_freq[LZFSE_ENCODE_D_SYMBOLS]; + uint16_t literal_freq[LZFSE_ENCODE_LITERAL_SYMBOLS]; +} lzfse_compressed_block_header_v1; + +/*! @abstract Compressed block header with compressed tables. Note that because + * freq[] is compressed, the structure-as-stored-in-the-stream is *truncated*; + * we only store the used bytes of freq[]. This means that some extra care must + * be taken when reading one of these headers from the stream. */ +typedef struct { + // Magic number, always LZFSE_COMPRESSEDV2_BLOCK_MAGIC. + uint32_t magic; + // Number of decoded (output) bytes in block. + uint32_t n_raw_bytes; + // The fields n_payload_bytes ... d_state from the + // lzfse_compressed_block_header_v1 object are packed into three 64-bit + // fields in the compressed header, as follows: + // + // offset bits value + // 0 20 n_literals + // 20 20 n_literal_payload_bytes + // 40 20 n_matches + // 60 3 literal_bits + // 63 1 --- unused --- + // + // 0 10 literal_state[0] + // 10 10 literal_state[1] + // 20 10 literal_state[2] + // 30 10 literal_state[3] + // 40 20 n_lmd_payload_bytes + // 60 3 lmd_bits + // 63 1 --- unused --- + // + // 0 32 header_size (total header size in bytes; this does not + // correspond to a field in the uncompressed header version, + // but is required; we wouldn't know the size of the + // compresssed header otherwise. + // 32 10 l_state + // 42 10 m_state + // 52 10 d_state + // 62 2 --- unused --- + uint64_t packed_fields[3]; + // Variable size freq tables, using a Huffman-style fixed encoding. + // Size allocated here is an upper bound (all values stored on 16 bits). + uint8_t freq[2 * (LZFSE_ENCODE_L_SYMBOLS + LZFSE_ENCODE_M_SYMBOLS + + LZFSE_ENCODE_D_SYMBOLS + LZFSE_ENCODE_LITERAL_SYMBOLS)]; +} __attribute__((__packed__, __aligned__(1))) +lzfse_compressed_block_header_v2; + +/*! @abstract LZVN compressed block header. */ +typedef struct { + // Magic number, always LZFSE_COMPRESSEDLZVN_BLOCK_MAGIC. + uint32_t magic; + // Number of decoded (output) bytes. + uint32_t n_raw_bytes; + // Number of encoded (source) bytes. + uint32_t n_payload_bytes; +} lzvn_compressed_block_header; + +// MARK: - LZFSE encode/decode interfaces + +int lzfse_encode_init(lzfse_encoder_state *s); +int lzfse_encode_translate(lzfse_encoder_state *s, lzfse_offset delta); +int lzfse_encode_base(lzfse_encoder_state *s); +int lzfse_encode_finish(lzfse_encoder_state *s); +int lzfse_decode(lzfse_decoder_state *s); + +// MARK: - LZVN encode/decode interfaces + +// Minimum source buffer size for compression. Smaller buffers will not be +// compressed; the lzvn encoder will simply return. +#define LZVN_ENCODE_MIN_SRC_SIZE ((size_t)8) + +// Maximum source buffer size for compression. Larger buffers will be +// compressed partially. +#define LZVN_ENCODE_MAX_SRC_SIZE ((size_t)0xffffffffU) + +// Minimum destination buffer size for compression. No compression will take +// place if smaller. +#define LZVN_ENCODE_MIN_DST_SIZE ((size_t)8) + +size_t lzvn_decode_scratch_size(void); +size_t lzvn_encode_scratch_size(void); +size_t lzvn_encode_buffer(void *__restrict dst, size_t dst_size, + const void *__restrict src, size_t src_size, + void *__restrict work); +size_t lzvn_decode_buffer(void *__restrict dst, size_t dst_size, + const void *__restrict src, size_t src_size); + +/*! @abstract Signed offset in buffers, stored on either 32 or 64 bits. */ +#if defined(_M_AMD64) || defined(__x86_64__) || defined(__arm64__) +typedef int64_t lzvn_offset; +#else +typedef int32_t lzvn_offset; +#endif + +// MARK: - LZFSE utility functions + +/*! @abstract Load bytes from memory location SRC. */ +LZFSE_INLINE uint16_t load2(const void *ptr) { + uint16_t data; + memcpy(&data, ptr, sizeof data); + return data; +} + +LZFSE_INLINE uint32_t load4(const void *ptr) { + uint32_t data; + memcpy(&data, ptr, sizeof data); + return data; +} + +LZFSE_INLINE uint64_t load8(const void *ptr) { + uint64_t data; + memcpy(&data, ptr, sizeof data); + return data; +} + +/*! @abstract Store bytes to memory location DST. */ +LZFSE_INLINE void store2(void *ptr, uint16_t data) { + memcpy(ptr, &data, sizeof data); +} + +LZFSE_INLINE void store4(void *ptr, uint32_t data) { + memcpy(ptr, &data, sizeof data); +} + +LZFSE_INLINE void store8(void *ptr, uint64_t data) { + memcpy(ptr, &data, sizeof data); +} + +/*! @abstract Load+store bytes from locations SRC to DST. Not intended for use + * with overlapping buffers. Note that for LZ-style compression, you need + * copies to behave like naive memcpy( ) implementations do, splatting the + * leading sequence if the buffers overlap. This copy does not do that, so + * should not be used with overlapping buffers. */ +LZFSE_INLINE void copy8(void *dst, const void *src) { store8(dst, load8(src)); } +LZFSE_INLINE void copy16(void *dst, const void *src) { + uint64_t m0 = load8(src); + uint64_t m1 = load8((const unsigned char *)src + 8); + store8(dst, m0); + store8((unsigned char *)dst + 8, m1); +} + +// =============================================================== +// Bitfield Operations + +/*! @abstract Extracts \p width bits from \p container, starting with \p lsb; if + * we view \p container as a bit array, we extract \c container[lsb:lsb+width]. */ +LZFSE_INLINE uintmax_t extract(uintmax_t container, unsigned lsb, + unsigned width) { + static const size_t container_width = sizeof container * 8; + assert(lsb < container_width); + assert(width > 0 && width <= container_width); + assert(lsb + width <= container_width); + if (width == container_width) + return container; + return (container >> lsb) & (((uintmax_t)1 << width) - 1); +} + +/*! @abstract Inserts \p width bits from \p data into \p container, starting with \p lsb. + * Viewed as bit arrays, the operations is: + * @code + * container[:lsb] is unchanged + * container[lsb:lsb+width] <-- data[0:width] + * container[lsb+width:] is unchanged + * @endcode + */ +LZFSE_INLINE uintmax_t insert(uintmax_t container, uintmax_t data, unsigned lsb, + unsigned width) { + static const size_t container_width = sizeof container * 8; + assert(lsb < container_width); + assert(width > 0 && width <= container_width); + assert(lsb + width <= container_width); + if (width == container_width) + return container; + uintmax_t mask = ((uintmax_t)1 << width) - 1; + return (container & ~(mask << lsb)) | (data & mask) << lsb; +} + +/*! @abstract Perform sanity checks on the values of lzfse_compressed_block_header_v1. + * Test that the field values are in the allowed limits, verify that the + * frequency tables sum to value less than total number of states. + * @return 0 if all tests passed. + * @return negative error code with 1 bit set for each failed test. */ +LZFSE_INLINE int lzfse_check_block_header_v1( + const lzfse_compressed_block_header_v1 *header) { + int tests_results = 0; + tests_results = + tests_results | + ((header->magic == LZFSE_COMPRESSEDV1_BLOCK_MAGIC) ? 0 : (1 << 0)); + tests_results = + tests_results | + ((header->n_literals <= LZFSE_LITERALS_PER_BLOCK) ? 0 : (1 << 1)); + tests_results = + tests_results | + ((header->n_matches <= LZFSE_MATCHES_PER_BLOCK) ? 0 : (1 << 2)); + + uint16_t literal_state[4]; + memcpy(literal_state, header->literal_state, sizeof(uint16_t) * 4); + + tests_results = + tests_results | + ((literal_state[0] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 3)); + tests_results = + tests_results | + ((literal_state[1] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 4)); + tests_results = + tests_results | + ((literal_state[2] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 5)); + tests_results = + tests_results | + ((literal_state[3] < LZFSE_ENCODE_LITERAL_STATES) ? 0 : (1 << 6)); + + tests_results = tests_results | + ((header->l_state < LZFSE_ENCODE_L_STATES) ? 0 : (1 << 7)); + tests_results = tests_results | + ((header->m_state < LZFSE_ENCODE_M_STATES) ? 0 : (1 << 8)); + tests_results = tests_results | + ((header->d_state < LZFSE_ENCODE_D_STATES) ? 0 : (1 << 9)); + + int res; + res = fse_check_freq(header->l_freq, LZFSE_ENCODE_L_SYMBOLS, + LZFSE_ENCODE_L_STATES); + tests_results = tests_results | ((res == 0) ? 0 : (1 << 10)); + res = fse_check_freq(header->m_freq, LZFSE_ENCODE_M_SYMBOLS, + LZFSE_ENCODE_M_STATES); + tests_results = tests_results | ((res == 0) ? 0 : (1 << 11)); + res = fse_check_freq(header->d_freq, LZFSE_ENCODE_D_SYMBOLS, + LZFSE_ENCODE_D_STATES); + tests_results = tests_results | ((res == 0) ? 0 : (1 << 12)); + res = fse_check_freq(header->literal_freq, LZFSE_ENCODE_LITERAL_SYMBOLS, + LZFSE_ENCODE_LITERAL_STATES); + tests_results = tests_results | ((res == 0) ? 0 : (1 << 13)); + + if (tests_results) { + return tests_results | 0x80000000; // each 1 bit is a test that failed + // (except for the sign bit) + } + + return 0; // OK +} + +// MARK: - L, M, D encoding constants for LZFSE + +// Largest encodable L (literal length), M (match length) and D (match +// distance) values. +#define LZFSE_ENCODE_MAX_L_VALUE 315 +#define LZFSE_ENCODE_MAX_M_VALUE 2359 +#define LZFSE_ENCODE_MAX_D_VALUE 262139 + +/*! @abstract The L, M, D data streams are all encoded as a "base" value, which is + * FSE-encoded, and an "extra bits" value, which is the difference between + * value and base, and is simply represented as a raw bit value (because it + * is the low-order bits of a larger number, not much entropy can be + * extracted from these bits by more complex encoding schemes). The following + * tables represent the number of low-order bits to encode separately and the + * base values for each of L, M, and D. + * + * @note The inverse tables for mapping the other way are significantly larger. + * Those tables have been split out to lzfse_encode_tables.h in order to keep + * this file relatively small. */ +static const uint8_t l_extra_bits[LZFSE_ENCODE_L_SYMBOLS] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3, 5, 8 +}; +static const int32_t l_base_value[LZFSE_ENCODE_L_SYMBOLS] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 28, 60 +}; +static const uint8_t m_extra_bits[LZFSE_ENCODE_M_SYMBOLS] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 8, 11 +}; +static const int32_t m_base_value[LZFSE_ENCODE_M_SYMBOLS] = { + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 56, 312 +}; +static const uint8_t d_extra_bits[LZFSE_ENCODE_D_SYMBOLS] = { + 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, + 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, + 8, 8, 8, 8, 9, 9, 9, 9, 10, 10, 10, 10, 11, 11, 11, 11, + 12, 12, 12, 12, 13, 13, 13, 13, 14, 14, 14, 14, 15, 15, 15, 15 +}; +static const int32_t d_base_value[LZFSE_ENCODE_D_SYMBOLS] = { + 0, 1, 2, 3, 4, 6, 8, 10, 12, 16, + 20, 24, 28, 36, 44, 52, 60, 76, 92, 108, + 124, 156, 188, 220, 252, 316, 380, 444, 508, 636, + 764, 892, 1020, 1276, 1532, 1788, 2044, 2556, 3068, 3580, + 4092, 5116, 6140, 7164, 8188, 10236, 12284, 14332, 16380, 20476, + 24572, 28668, 32764, 40956, 49148, 57340, 65532, 81916, 98300, 114684, + 131068, 163836, 196604, 229372 +}; + +#endif // LZFSE_INTERNAL_H diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_main.c b/view/kernelcache/core/transformers/liblzfse/lzfse_main.c new file mode 100644 index 00000000..dd4df99a --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_main.c @@ -0,0 +1,336 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZFSE command line tool + +#if !defined(_POSIX_C_SOURCE) || (_POSIX_C_SOURCE < 200112L) +# undef _POSIX_C_SOURCE +# define _POSIX_C_SOURCE 200112L +#endif + +#if defined(_MSC_VER) +# if !defined(_CRT_NONSTDC_NO_DEPRECATE) +# define _CRT_NONSTDC_NO_DEPRECATE +# endif +# if !defined(_CRT_SECURE_NO_WARNINGS) +# define _CRT_SECURE_NO_WARNINGS +# endif +# if !defined(__clang__) +# define inline __inline +# endif +#endif + +#include "lzfse.h" +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> + +#if defined(_MSC_VER) +# include <io.h> +# include <windows.h> +#else +# include <sys/time.h> +# include <unistd.h> +#endif + +// Same as realloc(x,s), except x is freed when realloc fails +static inline void *lzfse_reallocf(void *x, size_t s) { + void *y = realloc(x, s); + if (y == 0) { + free(x); + return 0; + } + return y; +} + +static double get_time() { +#if defined(_MSC_VER) + LARGE_INTEGER count, freq; + if (QueryPerformanceFrequency(&freq) && QueryPerformanceCounter(&count)) { + return (double)count.QuadPart / (double)freq.QuadPart; + } + return 1.0e-3 * (double)GetTickCount(); +#else + struct timeval tv; + if (gettimeofday(&tv, 0) != 0) { + perror("gettimeofday"); + exit(1); + } + return (double)tv.tv_sec + 1.0e-6 * (double)tv.tv_usec; +#endif +} + +//-------------------- + +enum { LZFSE_ENCODE = 0, LZFSE_DECODE }; + +void usage(int argc, char **argv) { + fprintf( + stderr, + "Usage: %s -encode|-decode [-i input_file] [-o output_file] [-h] [-v]\n", + argv[0]); +} + +#define USAGE(argc, argv) \ + do { \ + usage(argc, argv); \ + exit(0); \ + } while (0) +#define USAGE_MSG(argc, argv, ...) \ + do { \ + usage(argc, argv); \ + fprintf(stderr, __VA_ARGS__); \ + exit(1); \ + } while (0) + +int main(int argc, char **argv) { + const char *in_file = 0; // stdin + const char *out_file = 0; // stdout + int op = -1; // invalid op + int verbosity = 0; // quiet + + // Parse options + for (int i = 1; i < argc;) { + // no args + const char *a = argv[i++]; + if (strcmp(a, "-h") == 0) + USAGE(argc, argv); + if (strcmp(a, "-v") == 0) { + verbosity++; + continue; + } + if (strcmp(a, "-encode") == 0) { + op = LZFSE_ENCODE; + continue; + } + if (strcmp(a, "-decode") == 0) { + op = LZFSE_DECODE; + continue; + } + + // one arg + const char **arg_var = 0; + if (strcmp(a, "-i") == 0 && in_file == 0) + arg_var = &in_file; + else if (strcmp(a, "-o") == 0 && out_file == 0) + arg_var = &out_file; + if (arg_var != 0) { + // Flag is recognized. Check if there is an argument. + if (i == argc) + USAGE_MSG(argc, argv, "Error: Missing arg after %s\n", a); + *arg_var = argv[i++]; + continue; + } + + USAGE_MSG(argc, argv, "Error: invalid flag %s\n", a); + } + if (op < 0) + USAGE_MSG(argc, argv, "Error: -encode|-decode required\n"); + + // Info + if (verbosity > 0) { + if (op == LZFSE_ENCODE) + fprintf(stderr, "LZFSE encode\n"); + if (op == LZFSE_DECODE) + fprintf(stderr, "LZFSE decode\n"); + fprintf(stderr, "Input: %s\n", in_file ? in_file : "stdin"); + fprintf(stderr, "Output: %s\n", out_file ? out_file : "stdout"); + } + + // Load input + size_t in_allocated = 0; // allocated in IN + size_t in_size = 0; // used in IN + uint8_t *in = 0; // input buffer + int in_fd = -1; // input file desc + + if (in_file != 0) { + // If we have a file name, open it, and allocate the exact input size + struct stat st; +#if defined(_WIN32) + in_fd = open(in_file, O_RDONLY | O_BINARY); +#else + in_fd = open(in_file, O_RDONLY); +#endif + if (in_fd < 0) { + perror(in_file); + exit(1); + } + if (fstat(in_fd, &st) != 0) { + perror(in_file); + exit(1); + } + if (st.st_size > SIZE_MAX) { + fprintf(stderr, "File is too large\n"); + exit(1); + } + in_allocated = (size_t)st.st_size; + } else { + // Otherwise, read from stdin, and allocate to 1 MB, grow as needed + in_allocated = 1 << 20; + in_fd = 0; +#if defined(_WIN32) + if (setmode(in_fd, O_BINARY) == -1) { + perror("setmode"); + exit(1); + } +#endif + } + in = (uint8_t *)malloc(in_allocated); + if (in == 0) { + perror("malloc"); + exit(1); + } + + while (1) { + // re-alloc if needed + if (in_size == in_allocated) { + if (in_allocated < (100 << 20)) + in_allocated <<= 1; // double it + else + in_allocated += (100 << 20); // or add 100 MB if already large + in = lzfse_reallocf(in, in_allocated); + if (in == 0) { + perror("malloc"); + exit(1); + } + } + + ptrdiff_t r = read(in_fd, in + in_size, in_allocated - in_size); + if (r < 0) { + perror("read"); + exit(1); + } + if (r == 0) + break; // end of file + in_size += (size_t)r; + } + + if (in_file != 0) { + close(in_fd); + in_fd = -1; + } + + // Size info + if (verbosity > 0) { + fprintf(stderr, "Input size: %zu B\n", in_size); + } + + // Encode/decode + // Compute size for result buffer; we assume here that encode shrinks size, + // and that decode grows by no more than 4x. These are reasonable common- + // case guidelines, but are not formally guaranteed to be satisfied. + size_t out_allocated = (op == LZFSE_ENCODE) ? in_size : (4 * in_size); + size_t out_size = 0; + size_t aux_allocated = (op == LZFSE_ENCODE) ? lzfse_encode_scratch_size() + : lzfse_decode_scratch_size(); + void *aux = aux_allocated ? malloc(aux_allocated) : 0; + if (aux_allocated != 0 && aux == 0) { + perror("malloc"); + exit(1); + } + uint8_t *out = (uint8_t *)malloc(out_allocated); + if (out == 0) { + perror("malloc"); + exit(1); + } + + double c0 = get_time(); + while (1) { + if (op == LZFSE_ENCODE) + out_size = lzfse_encode_buffer(out, out_allocated, in, in_size, aux); + else + out_size = lzfse_decode_buffer(out, out_allocated, in, in_size, aux); + + // If output buffer was too small, grow and retry. + if (out_size == 0 || (op == LZFSE_DECODE && out_size == out_allocated)) { + if (verbosity > 0) + fprintf(stderr, "Output buffer was too small, increasing size...\n"); + out_allocated <<= 1; + out = (uint8_t *)lzfse_reallocf(out, out_allocated); + if (out == 0) { + perror("malloc"); + exit(1); + } + continue; + } + + break; + } + double c1 = get_time(); + + if (verbosity > 0) { + fprintf(stderr, "Output size: %zu B\n", out_size); + size_t raw_size = (op == LZFSE_ENCODE) ? in_size : out_size; + size_t compressed_size = (op == LZFSE_ENCODE) ? out_size : in_size; + fprintf(stderr, "Compression ratio: %.3f\n", + (double)raw_size / (double)compressed_size); + double ns_per_byte = 1.0e9 * (c1 - c0) / (double)raw_size; + double mb_per_s = (double)raw_size / 1024.0 / 1024.0 / (c1 - c0); + fprintf(stderr, "Speed: %.2f ns/B, %.2f MB/s\n",ns_per_byte,mb_per_s); + } + + // Write output + int out_fd = -1; + if (out_file) { +#if defined(_WIN32) + out_fd = open(out_file, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, + S_IWRITE); +#else + out_fd = open(out_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); +#endif + if (out_fd < 0) { + perror(out_file); + exit(1); + } + } else { + out_fd = 1; // stdout +#if defined(_WIN32) + if (setmode(out_fd, O_BINARY) == -1) { + perror("setmode"); + exit(1); + } +#endif + } + for (size_t out_pos = 0; out_pos < out_size;) { + ptrdiff_t w = write(out_fd, out + out_pos, out_size - out_pos); + if (w < 0) { + perror("write"); + exit(1); + } + if (w == 0) { + fprintf(stderr, "Failed to write to output file\n"); + exit(1); + } + out_pos += (size_t)w; + } + if (out_file != 0) { + close(out_fd); + out_fd = -1; + } + + free(in); + free(out); + free(aux); + return 0; // OK +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzfse_tunables.h b/view/kernelcache/core/transformers/liblzfse/lzfse_tunables.h new file mode 100644 index 00000000..4485a803 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzfse_tunables.h @@ -0,0 +1,60 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef LZFSE_TUNABLES_H +#define LZFSE_TUNABLES_H + +// Parameters controlling details of the LZ-style match search. These values +// may be modified to fine tune compression ratio vs. encoding speed, while +// keeping the compressed format compatible with LZFSE. Note that +// modifying them will also change the amount of work space required by +// the encoder. The values here are those used in the compression library +// on iOS and OS X. + +// Number of bits for hash function to produce. Should be in the range +// [10, 16]. Larger values reduce the number of false-positive found during +// the match search, and expand the history table, which may allow additional +// matches to be found, generally improving the achieved compression ratio. +// Larger values also increase the workspace size, and make it less likely +// that the history table will be present in cache, which reduces performance. +#define LZFSE_ENCODE_HASH_BITS 14 + +// Number of positions to store for each line in the history table. May +// be either 4 or 8. Using 8 doubles the size of the history table, which +// increases the chance of finding matches (thus improving compression ratio), +// but also increases the workspace size. +#define LZFSE_ENCODE_HASH_WIDTH 4 + +// Match length in bytes to cause immediate emission. Generally speaking, +// LZFSE maintains multiple candidate matches and waits to decide which match +// to emit until more information is available. When a match exceeds this +// threshold, it is emitted immediately. Thus, smaller values may give +// somewhat better performance, and larger values may give somewhat better +// compression ratios. +#define LZFSE_ENCODE_GOOD_MATCH 40 + +// When the source buffer is very small, LZFSE doesn't compress as well as +// some simpler algorithms. To maintain reasonable compression for these +// cases, we transition to use LZVN instead if the size of the source buffer +// is below this threshold. +#define LZFSE_ENCODE_LZVN_THRESHOLD 4096 + +#endif // LZFSE_TUNABLES_H diff --git a/view/kernelcache/core/transformers/liblzfse/lzvn_decode_base.c b/view/kernelcache/core/transformers/liblzfse/lzvn_decode_base.c new file mode 100644 index 00000000..c30180ec --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzvn_decode_base.c @@ -0,0 +1,711 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZVN low-level decoder + +#include "lzvn_decode_base.h" + +#if !defined(HAVE_LABELS_AS_VALUES) +# if defined(__GNUC__) || defined(__clang__) +# define HAVE_LABELS_AS_VALUES 1 +# else +# define HAVE_LABELS_AS_VALUES 0 +# endif +#endif + +// Both the source and destination buffers are represented by a pointer and +// a length; they are *always* updated in concert using this macro; however +// many bytes the pointer is advanced, the length is decremented by the same +// amount. Thus, pointer + length always points to the byte one past the end +// of the buffer. +#define PTR_LEN_INC(_pointer, _length, _increment) \ + (_pointer += _increment, _length -= _increment) + +// Update state with current positions and distance, corresponding to the +// beginning of an instruction in both streams +#define UPDATE_GOOD \ + (state->src = src_ptr, state->dst = dst_ptr, state->d_prev = D) + +void lzvn_decode(lzvn_decoder_state *state) { +#if HAVE_LABELS_AS_VALUES + // Jump table for all instructions + static const void *opc_tbl[256] = { + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&eos, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&nop, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&udef, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, + &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, &&med_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&sml_d, &&pre_d, &&lrg_d, + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, + &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, &&udef, + &&lrg_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, + &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, &&sml_l, + &&lrg_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, + &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m, &&sml_m}; +#endif + size_t src_len = state->src_end - state->src; + size_t dst_len = state->dst_end - state->dst; + if (src_len == 0 || dst_len == 0) + return; // empty buffer + + const unsigned char *src_ptr = state->src; + unsigned char *dst_ptr = state->dst; + size_t D = state->d_prev; + size_t M; + size_t L; + size_t opc_len; + + // Do we have a partially expanded match saved in state? + if (state->L != 0 || state->M != 0) { + L = state->L; + M = state->M; + D = state->D; + opc_len = 0; // we already skipped the op + state->L = state->M = state->D = 0; + if (M == 0) + goto copy_literal; + if (L == 0) + goto copy_match; + goto copy_literal_and_match; + } + + unsigned char opc = src_ptr[0]; + +#if HAVE_LABELS_AS_VALUES + goto *opc_tbl[opc]; +#else + for (;;) { + switch (opc) { +#endif +// =============================================================== +// These four opcodes (sml_d, med_d, lrg_d, and pre_d) encode both a +// literal and a match. The bulk of their implementations are shared; +// each label here only does the work of setting the opcode length (not +// including any literal bytes), and extracting the literal length, match +// length, and match distance (except in pre_d). They then jump into the +// shared implementation to actually output the literal and match bytes. +// +// No error checking happens in the first stage, except for ensuring that +// the source has enough length to represent the full opcode before +// reading past the first byte. +sml_d: +#if !HAVE_LABELS_AS_VALUES + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 8: + case 9: + case 10: + case 11: + case 12: + case 13: + case 16: + case 17: + case 18: + case 19: + case 20: + case 21: + case 24: + case 25: + case 26: + case 27: + case 28: + case 29: + case 32: + case 33: + case 34: + case 35: + case 36: + case 37: + case 40: + case 41: + case 42: + case 43: + case 44: + case 45: + case 48: + case 49: + case 50: + case 51: + case 52: + case 53: + case 56: + case 57: + case 58: + case 59: + case 60: + case 61: + case 64: + case 65: + case 66: + case 67: + case 68: + case 69: + case 72: + case 73: + case 74: + case 75: + case 76: + case 77: + case 80: + case 81: + case 82: + case 83: + case 84: + case 85: + case 88: + case 89: + case 90: + case 91: + case 92: + case 93: + case 96: + case 97: + case 98: + case 99: + case 100: + case 101: + case 104: + case 105: + case 106: + case 107: + case 108: + case 109: + case 128: + case 129: + case 130: + case 131: + case 132: + case 133: + case 136: + case 137: + case 138: + case 139: + case 140: + case 141: + case 144: + case 145: + case 146: + case 147: + case 148: + case 149: + case 152: + case 153: + case 154: + case 155: + case 156: + case 157: + case 192: + case 193: + case 194: + case 195: + case 196: + case 197: + case 200: + case 201: + case 202: + case 203: + case 204: + case 205: +#endif + UPDATE_GOOD; + // "small distance": This opcode has the structure LLMMMDDD DDDDDDDD LITERAL + // where the length of literal (0-3 bytes) is encoded by the high 2 bits of + // the first byte. We first extract the literal length so we know how long + // the opcode is, then check that the source can hold both this opcode and + // at least one byte of the next (because any valid input stream must be + // terminated with an eos token). + opc_len = 2; + L = (size_t)extract(opc, 6, 2); + M = (size_t)extract(opc, 3, 3) + 3; + // We need to ensure that the source buffer is long enough that we can + // safely read this entire opcode, the literal that follows, and the first + // byte of the next opcode. Once we satisfy this requirement, we can + // safely unpack the match distance. A check similar to this one is + // present in all the opcode implementations. + if (src_len <= opc_len + L) + return; // source truncated + D = (size_t)extract(opc, 0, 3) << 8 | src_ptr[1]; + goto copy_literal_and_match; + +med_d: +#if !HAVE_LABELS_AS_VALUES + case 160: + case 161: + case 162: + case 163: + case 164: + case 165: + case 166: + case 167: + case 168: + case 169: + case 170: + case 171: + case 172: + case 173: + case 174: + case 175: + case 176: + case 177: + case 178: + case 179: + case 180: + case 181: + case 182: + case 183: + case 184: + case 185: + case 186: + case 187: + case 188: + case 189: + case 190: + case 191: +#endif + UPDATE_GOOD; + // "medium distance": This is a minor variant of the "small distance" + // encoding, where we will now use two extra bytes instead of one to encode + // the restof the match length and distance. This allows an extra two bits + // for the match length, and an extra three bits for the match distance. The + // full structure of the opcode is 101LLMMM DDDDDDMM DDDDDDDD LITERAL. + opc_len = 3; + L = (size_t)extract(opc, 3, 2); + if (src_len <= opc_len + L) + return; // source truncated + uint16_t opc23 = load2(&src_ptr[1]); + M = (size_t)((extract(opc, 0, 3) << 2 | extract(opc23, 0, 2)) + 3); + D = (size_t)extract(opc23, 2, 14); + goto copy_literal_and_match; + +lrg_d: +#if !HAVE_LABELS_AS_VALUES + case 7: + case 15: + case 23: + case 31: + case 39: + case 47: + case 55: + case 63: + case 71: + case 79: + case 87: + case 95: + case 103: + case 111: + case 135: + case 143: + case 151: + case 159: + case 199: + case 207: +#endif + UPDATE_GOOD; + // "large distance": This is another variant of the "small distance" + // encoding, where we will now use two extra bytes to encode the match + // distance, which allows distances up to 65535 to be represented. The full + // structure of the opcode is LLMMM111 DDDDDDDD DDDDDDDD LITERAL. + opc_len = 3; + L = (size_t)extract(opc, 6, 2); + M = (size_t)extract(opc, 3, 3) + 3; + if (src_len <= opc_len + L) + return; // source truncated + D = load2(&src_ptr[1]); + goto copy_literal_and_match; + +pre_d: +#if !HAVE_LABELS_AS_VALUES + case 70: + case 78: + case 86: + case 94: + case 102: + case 110: + case 134: + case 142: + case 150: + case 158: + case 198: + case 206: +#endif + UPDATE_GOOD; + // "previous distance": This opcode has the structure LLMMM110, where the + // length of the literal (0-3 bytes) is encoded by the high 2 bits of the + // first byte. We first extract the literal length so we know how long + // the opcode is, then check that the source can hold both this opcode and + // at least one byte of the next (because any valid input stream must be + // terminated with an eos token). + opc_len = 1; + L = (size_t)extract(opc, 6, 2); + M = (size_t)extract(opc, 3, 3) + 3; + if (src_len <= opc_len + L) + return; // source truncated + goto copy_literal_and_match; + +copy_literal_and_match: + // Common implementation of writing data for opcodes that have both a + // literal and a match. We begin by advancing the source pointer past + // the opcode, so that it points at the first literal byte (if L + // is non-zero; otherwise it points at the next opcode). + PTR_LEN_INC(src_ptr, src_len, opc_len); + // Now we copy the literal from the source pointer to the destination. + if (__builtin_expect(dst_len >= 4 && src_len >= 4, 1)) { + // The literal is 0-3 bytes; if we are not near the end of the buffer, + // we can safely just do a 4 byte copy (which is guaranteed to cover + // the complete literal, and may include some other bytes as well). + store4(dst_ptr, load4(src_ptr)); + } else if (L <= dst_len) { + // We are too close to the end of either the input or output stream + // to be able to safely use a four-byte copy, but we will not exhaust + // either stream (we already know that the source will not be + // exhausted from checks in the individual opcode implementations, + // and we just tested that dst_len > L). Thus, we need to do a + // byte-by-byte copy of the literal. This is slow, but it can only ever + // happen near the very end of a buffer, so it is not an important case to + // optimize. + for (size_t i = 0; i < L; ++i) + dst_ptr[i] = src_ptr[i]; + } else { + // Destination truncated: fill DST, and store partial match + + // Copy partial literal + for (size_t i = 0; i < dst_len; ++i) + dst_ptr[i] = src_ptr[i]; + // Save state + state->src = src_ptr + dst_len; + state->dst = dst_ptr + dst_len; + state->L = L - dst_len; + state->M = M; + state->D = D; + return; // destination truncated + } + // Having completed the copy of the literal, we advance both the source + // and destination pointers by the number of literal bytes. + PTR_LEN_INC(dst_ptr, dst_len, L); + PTR_LEN_INC(src_ptr, src_len, L); + // Check if the match distance is valid; matches must not reference + // bytes that preceed the start of the output buffer, nor can the match + // distance be zero. + if (D > dst_ptr - state->dst_begin || D == 0) + goto invalid_match_distance; +copy_match: + // Now we copy the match from dst_ptr - D to dst_ptr. It is important to keep + // in mind that we may have D < M, in which case the source and destination + // windows overlap in the copy. The semantics of the match copy are *not* + // those of memmove( ); if the buffers overlap it needs to behave as though + // we were copying byte-by-byte in increasing address order. If, for example, + // D is 1, the copy operation is equivalent to: + // + // memset(dst_ptr, dst_ptr[-1], M); + // + // i.e. it splats the previous byte. This means that we need to be very + // careful about using wide loads or stores to perform the copy operation. + if (__builtin_expect(dst_len >= M + 7 && D >= 8, 1)) { + // We are not near the end of the buffer, and the match distance + // is at least eight. Thus, we can safely loop using eight byte + // copies. The last of these may slop over the intended end of + // the match, but this is OK because we know we have a safety bound + // away from the end of the destination buffer. + for (size_t i = 0; i < M; i += 8) + store8(&dst_ptr[i], load8(&dst_ptr[i - D])); + } else if (M <= dst_len) { + // Either the match distance is too small, or we are too close to + // the end of the buffer to safely use eight byte copies. Fall back + // on a simple byte-by-byte implementation. + for (size_t i = 0; i < M; ++i) + dst_ptr[i] = dst_ptr[i - D]; + } else { + // Destination truncated: fill DST, and store partial match + + // Copy partial match + for (size_t i = 0; i < dst_len; ++i) + dst_ptr[i] = dst_ptr[i - D]; + // Save state + state->src = src_ptr; + state->dst = dst_ptr + dst_len; + state->L = 0; + state->M = M - dst_len; + state->D = D; + return; // destination truncated + } + // Update the destination pointer and length to account for the bytes + // written by the match, then load the next opcode byte and branch to + // the appropriate implementation. + PTR_LEN_INC(dst_ptr, dst_len, M); + opc = src_ptr[0]; +#if HAVE_LABELS_AS_VALUES + goto *opc_tbl[opc]; +#else + break; +#endif + +// =============================================================== +// Opcodes representing only a match (no literal). +// These two opcodes (lrg_m and sml_m) encode only a match. The match +// distance is carried over from the previous opcode, so all they need +// to encode is the match length. We are able to reuse the match copy +// sequence from the literal and match opcodes to perform the actual +// copy implementation. +sml_m: +#if !HAVE_LABELS_AS_VALUES + case 241: + case 242: + case 243: + case 244: + case 245: + case 246: + case 247: + case 248: + case 249: + case 250: + case 251: + case 252: + case 253: + case 254: + case 255: +#endif + UPDATE_GOOD; + // "small match": This opcode has no literal, and uses the previous match + // distance (i.e. it encodes only the match length), in a single byte as + // 1111MMMM. + opc_len = 1; + if (src_len <= opc_len) + return; // source truncated + M = (size_t)extract(opc, 0, 4); + PTR_LEN_INC(src_ptr, src_len, opc_len); + goto copy_match; + +lrg_m: +#if !HAVE_LABELS_AS_VALUES + case 240: +#endif + UPDATE_GOOD; + // "large match": This opcode has no literal, and uses the previous match + // distance (i.e. it encodes only the match length). It is encoded in two + // bytes as 11110000 MMMMMMMM. Because matches smaller than 16 bytes can + // be represented by sml_m, there is an implicit bias of 16 on the match + // length; the representable values are [16,271]. + opc_len = 2; + if (src_len <= opc_len) + return; // source truncated + M = src_ptr[1] + 16; + PTR_LEN_INC(src_ptr, src_len, opc_len); + goto copy_match; + +// =============================================================== +// Opcodes representing only a literal (no match). +// These two opcodes (lrg_l and sml_l) encode only a literal. There is no +// match length or match distance to worry about (but we need to *not* +// touch D, as it must be preserved between opcodes). +sml_l: +#if !HAVE_LABELS_AS_VALUES + case 225: + case 226: + case 227: + case 228: + case 229: + case 230: + case 231: + case 232: + case 233: + case 234: + case 235: + case 236: + case 237: + case 238: + case 239: +#endif + UPDATE_GOOD; + // "small literal": This opcode has no match, and encodes only a literal + // of length up to 15 bytes. The format is 1110LLLL LITERAL. + opc_len = 1; + L = (size_t)extract(opc, 0, 4); + goto copy_literal; + +lrg_l: +#if !HAVE_LABELS_AS_VALUES + case 224: +#endif + UPDATE_GOOD; + // "large literal": This opcode has no match, and uses the previous match + // distance (i.e. it encodes only the match length). It is encoded in two + // bytes as 11100000 LLLLLLLL LITERAL. Because literals smaller than 16 + // bytes can be represented by sml_l, there is an implicit bias of 16 on + // the literal length; the representable values are [16,271]. + opc_len = 2; + if (src_len <= 2) + return; // source truncated + L = src_ptr[1] + 16; + goto copy_literal; + +copy_literal: + // Check that the source buffer is large enough to hold the complete + // literal and at least the first byte of the next opcode. If so, advance + // the source pointer to point to the first byte of the literal and adjust + // the source length accordingly. + if (src_len <= opc_len + L) + return; // source truncated + PTR_LEN_INC(src_ptr, src_len, opc_len); + // Now we copy the literal from the source pointer to the destination. + if (dst_len >= L + 7 && src_len >= L + 7) { + // We are not near the end of the source or destination buffers; thus + // we can safely copy the literal using wide copies, without worrying + // about reading or writing past the end of either buffer. + for (size_t i = 0; i < L; i += 8) + store8(&dst_ptr[i], load8(&src_ptr[i])); + } else if (L <= dst_len) { + // We are too close to the end of either the input or output stream + // to be able to safely use an eight-byte copy. Instead we copy the + // literal byte-by-byte. + for (size_t i = 0; i < L; ++i) + dst_ptr[i] = src_ptr[i]; + } else { + // Destination truncated: fill DST, and store partial match + + // Copy partial literal + for (size_t i = 0; i < dst_len; ++i) + dst_ptr[i] = src_ptr[i]; + // Save state + state->src = src_ptr + dst_len; + state->dst = dst_ptr + dst_len; + state->L = L - dst_len; + state->M = 0; + state->D = D; + return; // destination truncated + } + // Having completed the copy of the literal, we advance both the source + // and destination pointers by the number of literal bytes. + PTR_LEN_INC(dst_ptr, dst_len, L); + PTR_LEN_INC(src_ptr, src_len, L); + // Load the first byte of the next opcode, and jump to its implementation. + opc = src_ptr[0]; +#if HAVE_LABELS_AS_VALUES + goto *opc_tbl[opc]; +#else + break; +#endif + +// =============================================================== +// Other opcodes +nop: +#if !HAVE_LABELS_AS_VALUES + case 14: + case 22: +#endif + UPDATE_GOOD; + opc_len = 1; + if (src_len <= opc_len) + return; // source truncated + PTR_LEN_INC(src_ptr, src_len, opc_len); + opc = src_ptr[0]; +#if HAVE_LABELS_AS_VALUES + goto *opc_tbl[opc]; +#else + break; +#endif + +eos: +#if !HAVE_LABELS_AS_VALUES + case 6: +#endif + opc_len = 8; + if (src_len < opc_len) + return; // source truncated (here we don't need an extra byte for next op + // code) + PTR_LEN_INC(src_ptr, src_len, opc_len); + state->end_of_stream = 1; + UPDATE_GOOD; + return; // end-of-stream + +// =============================================================== +// Return on error +udef: +#if !HAVE_LABELS_AS_VALUES + case 30: + case 38: + case 46: + case 54: + case 62: + case 112: + case 113: + case 114: + case 115: + case 116: + case 117: + case 118: + case 119: + case 120: + case 121: + case 122: + case 123: + case 124: + case 125: + case 126: + case 127: + case 208: + case 209: + case 210: + case 211: + case 212: + case 213: + case 214: + case 215: + case 216: + case 217: + case 218: + case 219: + case 220: + case 221: + case 222: + case 223: +#endif +invalid_match_distance: + + return; // we already updated state +#if !HAVE_LABELS_AS_VALUES + } + } +#endif +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzvn_decode_base.h b/view/kernelcache/core/transformers/liblzfse/lzvn_decode_base.h new file mode 100644 index 00000000..b4307a53 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzvn_decode_base.h @@ -0,0 +1,68 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZVN low-level decoder (v2) +// Functions in the low-level API should switch to these at some point. +// Apr 2014 + +#ifndef LZVN_DECODE_BASE_H +#define LZVN_DECODE_BASE_H + +#include "lzfse_internal.h" + +/*! @abstract Base decoder state. */ +typedef struct { + + // Decoder I/O + + // Next byte to read in source buffer + const unsigned char *src; + // Next byte after source buffer + const unsigned char *src_end; + + // Next byte to write in destination buffer (by decoder) + unsigned char *dst; + // Valid range for destination buffer is [dst_begin, dst_end - 1] + unsigned char *dst_begin; + unsigned char *dst_end; + // Next byte to read in destination buffer (modified by caller) + unsigned char *dst_current; + + // Decoder state + + // Partially expanded match, or 0,0,0. + // In that case, src points to the next literal to copy, or the next op-code + // if L==0. + size_t L, M, D; + + // Distance for last emitted match, or 0 + lzvn_offset d_prev; + + // Did we decode end-of-stream? + int end_of_stream; + +} lzvn_decoder_state; + +/*! @abstract Decode source to destination. + * Updates \p state (src,dst,d_prev). */ +void lzvn_decode(lzvn_decoder_state *state); + +#endif // LZVN_DECODE_BASE_H diff --git a/view/kernelcache/core/transformers/liblzfse/lzvn_encode_base.c b/view/kernelcache/core/transformers/liblzfse/lzvn_encode_base.c new file mode 100644 index 00000000..c86b5511 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzvn_encode_base.c @@ -0,0 +1,593 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZVN low-level encoder + +#include "lzvn_encode_base.h" + +#if defined(_MSC_VER) && !defined(__clang__) +# define restrict __restrict +#endif + +// =============================================================== +// Coarse/fine copy, non overlapping buffers + +/*! @abstract Copy at least \p nbytes bytes from \p src to \p dst, by blocks + * of 8 bytes (may go beyond range). No overlap. + * @return \p dst + \p nbytes. */ +static inline unsigned char *lzvn_copy64(unsigned char *restrict dst, + const unsigned char *restrict src, + size_t nbytes) { + for (size_t i = 0; i < nbytes; i += 8) + store8(dst + i, load8(src + i)); + return dst + nbytes; +} + +/*! @abstract Copy exactly \p nbytes bytes from \p src to \p dst (respects range). + * No overlap. + * @return \p dst + \p nbytes. */ +static inline unsigned char *lzvn_copy8(unsigned char *restrict dst, + const unsigned char *restrict src, + size_t nbytes) { + for (size_t i = 0; i < nbytes; i++) + dst[i] = src[i]; + return dst + nbytes; +} + +/*! @abstract Emit (L,0,0) instructions (final literal). + * We read at most \p L bytes from \p p. + * @param p input stream + * @param q1 the first byte after the output buffer. + * @return pointer to the next output, <= \p q1. + * @return \p q1 if output is full. In that case, output will be partially invalid. + */ +static inline unsigned char *emit_literal(const unsigned char *p, + unsigned char *q, unsigned char *q1, + size_t L) { + size_t x; + while (L > 15) { + x = L < 271 ? L : 271; + if (q + x + 10 >= q1) + goto OUT_FULL; + store2(q, 0xE0 + ((x - 16) << 8)); + q += 2; + L -= x; + q = lzvn_copy8(q, p, x); + p += x; + } + if (L > 0) { + if (q + L + 10 >= q1) + goto OUT_FULL; + *q++ = 0xE0 + L; // 1110LLLL + q = lzvn_copy8(q, p, L); + } + return q; + +OUT_FULL: + return q1; +} + +/*! @abstract Emit (L,M,D) instructions. M>=3. + * @param p input stream pointing to the beginning of the literal. We read at + * most \p L+4 bytes from \p p. + * @param q1 the first byte after the output buffer. + * @return pointer to the next output, <= \p q1. + * @return \p q1 if output is full. In that case, output will be partially invalid. + */ +static inline unsigned char *emit(const unsigned char *p, unsigned char *q, + unsigned char *q1, size_t L, size_t M, + size_t D, size_t D_prev) { + size_t x; + while (L > 15) { + x = L < 271 ? L : 271; + if (q + x + 10 >= q1) + goto OUT_FULL; + store2(q, 0xE0 + ((x - 16) << 8)); + q += 2; + L -= x; + q = lzvn_copy64(q, p, x); + p += x; + } + if (L > 3) { + if (q + L + 10 >= q1) + goto OUT_FULL; + *q++ = 0xE0 + L; // 1110LLLL + q = lzvn_copy64(q, p, L); + p += L; + L = 0; + } + x = M <= 10 - 2 * L ? M : 10 - 2 * L; // x = min(10-2*L,M) + M -= x; + x -= 3; // M = (x+3) + M' max value for x is 7-2*L + + // Here L<4 literals remaining, we read them here + uint32_t literal = load4(p); + // P is not accessed after this point + + // Relaxed capacity test covering all cases + if (q + 8 >= q1) + goto OUT_FULL; + + if (D == D_prev) { + if (L == 0) { + *q++ = 0xF0 + (x + 3); // XM! + } else { + *q++ = (L << 6) + (x << 3) + 6; // LLxxx110 + } + store4(q, literal); + q += L; + } else if (D < 2048 - 2 * 256) { + // Short dist D>>8 in 0..5 + *q++ = (D >> 8) + (L << 6) + (x << 3); // LLxxxDDD + *q++ = D & 0xFF; + store4(q, literal); + q += L; + } else if (D >= (1 << 14) || M == 0 || (x + 3) + M > 34) { + // Long dist + *q++ = (L << 6) + (x << 3) + 7; + store2(q, D); + q += 2; + store4(q, literal); + q += L; + } else { + // Medium distance + x += M; + M = 0; + *q++ = 0xA0 + (x >> 2) + (L << 3); + store2(q, D << 2 | (x & 3)); + q += 2; + store4(q, literal); + q += L; + } + + // Issue remaining match + while (M > 15) { + if (q + 2 >= q1) + goto OUT_FULL; + x = M < 271 ? M : 271; + store2(q, 0xf0 + ((x - 16) << 8)); + q += 2; + M -= x; + } + if (M > 0) { + if (q + 1 >= q1) + goto OUT_FULL; + *q++ = 0xF0 + M; // M = 0..15 + } + + return q; + +OUT_FULL: + return q1; +} + +// =============================================================== +// Conversions + +/*! @abstract Return 32-bit value to store for offset x. */ +static inline int32_t offset_to_s32(lzvn_offset x) { return (int32_t)x; } + +/*! @abstract Get offset from 32-bit stored value x. */ +static inline lzvn_offset offset_from_s32(int32_t x) { return (lzvn_offset)x; } + +// =============================================================== +// Hash and Matching + +/*! @abstract Get hash in range \c [0,LZVN_ENCODE_HASH_VALUES-1] from 3 bytes in i. */ +static inline uint32_t hash3i(uint32_t i) { + i &= 0xffffff; // truncate to 24-bit input (slightly increases compression ratio) + uint32_t h = (i * (1 + (1 << 6) + (1 << 12))) >> 12; + return h & (LZVN_ENCODE_HASH_VALUES - 1); +} + +/*! @abstract Return the number [0, 4] of zero bytes in \p x, starting from the + * least significant byte. */ +static inline lzvn_offset trailing_zero_bytes(uint32_t x) { + return (x == 0) ? 4 : (__builtin_ctzl(x) >> 3); +} + +/*! @abstract Return the number [0, 4] of matching chars between values at + * \p src+i and \p src+j, starting from the least significant byte. + * Assumes we can read 4 chars from each position. */ +static inline lzvn_offset nmatch4(const unsigned char *src, lzvn_offset i, + lzvn_offset j) { + uint32_t vi = load4(src + i); + uint32_t vj = load4(src + j); + return trailing_zero_bytes(vi ^ vj); +} + +/*! @abstract Check if l_begin, m_begin, m0_begin (m0_begin < m_begin) can be + * expanded to a match of length at least 3. + * @param m_begin new string to match. + * @param m0_begin candidate old string. + * @param src source buffer, with valid indices src_begin <= i < src_end. + * (src_begin may be <0) + * @return If a match can be found, return 1 and set all \p match fields, + * otherwise return 0. + * @note \p *match should be 0 before the call. */ +static inline int lzvn_find_match(const unsigned char *src, + lzvn_offset src_begin, + lzvn_offset src_end, lzvn_offset l_begin, + lzvn_offset m0_begin, lzvn_offset m_begin, + lzvn_match_info *match) { + lzvn_offset n = nmatch4(src, m_begin, m0_begin); + if (n < 3) + return 0; // no match + + lzvn_offset D = m_begin - m0_begin; // actual distance + if (D <= 0 || D > LZVN_ENCODE_MAX_DISTANCE) + return 0; // distance out of range + + // Expand forward + lzvn_offset m_end = m_begin + n; + while (n == 4 && m_end + 4 < src_end) { + n = nmatch4(src, m_end, m_end - D); + m_end += n; + } + + // Expand backwards over literal + while (m0_begin > src_begin && m_begin > l_begin && + src[m_begin - 1] == src[m0_begin - 1]) { + m0_begin--; + m_begin--; + } + + // OK, we keep it, update MATCH + lzvn_offset M = m_end - m_begin; // match length + match->m_begin = m_begin; + match->m_end = m_end; + match->K = M - ((D < 0x600) ? 2 : 3); + match->M = M; + match->D = D; + + return 1; // OK +} + +/*! @abstract Same as lzvn_find_match, but we already know that N bytes do + * match (N<=4). */ +static inline int lzvn_find_matchN(const unsigned char *src, + lzvn_offset src_begin, + lzvn_offset src_end, lzvn_offset l_begin, + lzvn_offset m0_begin, lzvn_offset m_begin, + lzvn_offset n, lzvn_match_info *match) { + // We can skip the first comparison on 4 bytes + if (n < 3) + return 0; // no match + + lzvn_offset D = m_begin - m0_begin; // actual distance + if (D <= 0 || D > LZVN_ENCODE_MAX_DISTANCE) + return 0; // distance out of range + + // Expand forward + lzvn_offset m_end = m_begin + n; + while (n == 4 && m_end + 4 < src_end) { + n = nmatch4(src, m_end, m_end - D); + m_end += n; + } + + // Expand backwards over literal + while (m0_begin > src_begin && m_begin > l_begin && + src[m_begin - 1] == src[m0_begin - 1]) { + m0_begin--; + m_begin--; + } + + // OK, we keep it, update MATCH + lzvn_offset M = m_end - m_begin; // match length + match->m_begin = m_begin; + match->m_end = m_end; + match->K = M - ((D < 0x600) ? 2 : 3); + match->M = M; + match->D = D; + + return 1; // OK +} + +// =============================================================== +// Encoder Backend + +/*! @abstract Emit a match and update state. + * @return number of bytes written to \p dst. May be 0 if there is no more space + * in \p dst to emit the match. */ +static inline lzvn_offset lzvn_emit_match(lzvn_encoder_state *state, + lzvn_match_info match) { + size_t L = (size_t)(match.m_begin - state->src_literal); // literal count + size_t M = (size_t)match.M; // match length + size_t D = (size_t)match.D; // match distance + size_t D_prev = (size_t)state->d_prev; // previously emitted match distance + unsigned char *dst = emit(state->src + state->src_literal, state->dst, + state->dst_end, L, M, D, D_prev); + // Check if DST is full + if (dst >= state->dst_end) { + return 0; // FULL + } + + // Update state + lzvn_offset dst_used = dst - state->dst; + state->d_prev = match.D; + state->dst = dst; + state->src_literal = match.m_end; + return dst_used; +} + +/*! @abstract Emit a n-bytes literal and update state. + * @return number of bytes written to \p dst. May be 0 if there is no more space + * in \p dst to emit the literal. */ +static inline lzvn_offset lzvn_emit_literal(lzvn_encoder_state *state, + lzvn_offset n) { + size_t L = (size_t)n; + unsigned char *dst = emit_literal(state->src + state->src_literal, state->dst, + state->dst_end, L); + // Check if DST is full + if (dst >= state->dst_end) + return 0; // FULL + + // Update state + lzvn_offset dst_used = dst - state->dst; + state->dst = dst; + state->src_literal += n; + return dst_used; +} + +/*! @abstract Emit end-of-stream and update state. + * @return number of bytes written to \p dst. May be 0 if there is no more space + * in \p dst to emit the instruction. */ +static inline lzvn_offset lzvn_emit_end_of_stream(lzvn_encoder_state *state) { + // Do we have 8 byte in dst? + if (state->dst_end < state->dst + 8) + return 0; // FULL + + // Insert end marker and update state + store8(state->dst, 0x06); // end-of-stream command + state->dst += 8; + return 8; // dst_used +} + +// =============================================================== +// Encoder Functions + +/*! @abstract Initialize encoder table in \p state, uses current I/O parameters. */ +static inline void lzvn_init_table(lzvn_encoder_state *state) { + lzvn_offset index = -LZVN_ENCODE_MAX_DISTANCE; // max match distance + if (index < state->src_begin) + index = state->src_begin; + uint32_t value = load4(state->src + index); + + lzvn_encode_entry_type e; + for (int i = 0; i < 4; i++) { + e.indices[i] = offset_to_s32(index); + e.values[i] = value; + } + for (int u = 0; u < LZVN_ENCODE_HASH_VALUES; u++) + state->table[u] = e; // fill entire table +} + +void lzvn_encode(lzvn_encoder_state *state) { + const lzvn_match_info NO_MATCH = {0}; + + for (; state->src_current < state->src_current_end; state->src_current++) { + // Get 4 bytes at src_current + uint32_t vi = load4(state->src + state->src_current); + + // Compute new hash H at position I, and push value into position table + int h = hash3i(vi); // index of first entry + + // Read table entries for H + lzvn_encode_entry_type e = state->table[h]; + + // Update entry with index=current and value=vi + lzvn_encode_entry_type updated_e; // rotate values, so we will replace the oldest + updated_e.indices[0] = offset_to_s32(state->src_current); + updated_e.indices[1] = e.indices[0]; + updated_e.indices[2] = e.indices[1]; + updated_e.indices[3] = e.indices[2]; + updated_e.values[0] = vi; + updated_e.values[1] = e.values[0]; + updated_e.values[2] = e.values[1]; + updated_e.values[3] = e.values[2]; + + // Do not check matches if still in previously emitted match + if (state->src_current < state->src_literal) + goto after_emit; + +// Update best with candidate if better +#define UPDATE(best, candidate) \ + do { \ + if (candidate.K > best.K || \ + ((candidate.K == best.K) && (candidate.m_end > best.m_end + 1))) { \ + best = candidate; \ + } \ + } while (0) +// Check candidate. Keep if better. +#define CHECK_CANDIDATE(ik, nk) \ + do { \ + lzvn_match_info m1; \ + if (lzvn_find_matchN(state->src, state->src_begin, state->src_end, \ + state->src_literal, ik, state->src_current, nk, &m1)) { \ + UPDATE(incoming, m1); \ + } \ + } while (0) +// Emit match M. Return if we don't have enough space in the destination buffer +#define EMIT_MATCH(m) \ + do { \ + if (lzvn_emit_match(state, m) == 0) \ + return; \ + } while (0) +// Emit literal of length L. Return if we don't have enough space in the +// destination buffer +#define EMIT_LITERAL(l) \ + do { \ + if (lzvn_emit_literal(state, l) == 0) \ + return; \ + } while (0) + + lzvn_match_info incoming = NO_MATCH; + + // Check candidates in order (closest first) + uint32_t diffs[4]; + for (int k = 0; k < 4; k++) + diffs[k] = e.values[k] ^ vi; // XOR, 0 if equal + lzvn_offset ik; // index + lzvn_offset nk; // match byte count + + // The values stored in e.xyzw are 32-bit signed indices, extended to signed + // type lzvn_offset + ik = offset_from_s32(e.indices[0]); + nk = trailing_zero_bytes(diffs[0]); + CHECK_CANDIDATE(ik, nk); + ik = offset_from_s32(e.indices[1]); + nk = trailing_zero_bytes(diffs[1]); + CHECK_CANDIDATE(ik, nk); + ik = offset_from_s32(e.indices[2]); + nk = trailing_zero_bytes(diffs[2]); + CHECK_CANDIDATE(ik, nk); + ik = offset_from_s32(e.indices[3]); + nk = trailing_zero_bytes(diffs[3]); + CHECK_CANDIDATE(ik, nk); + + // Check candidate at previous distance + if (state->d_prev != 0) { + lzvn_match_info m1; + if (lzvn_find_match(state->src, state->src_begin, state->src_end, + state->src_literal, state->src_current - state->d_prev, + state->src_current, &m1)) { + m1.K = m1.M - 1; // fix K for D_prev + UPDATE(incoming, m1); + } + } + + // Here we have the best candidate in incoming, may be NO_MATCH + + // If no incoming match, and literal backlog becomes too high, emit pending + // match, or literals if there is no pending match + if (incoming.M == 0) { + if (state->src_current - state->src_literal >= + LZVN_ENCODE_MAX_LITERAL_BACKLOG) // at this point, we always have + // current >= literal + { + if (state->pending.M != 0) { + EMIT_MATCH(state->pending); + state->pending = NO_MATCH; + } else { + EMIT_LITERAL(271); // emit long literal (271 is the longest literal size we allow) + } + } + goto after_emit; + } + + if (state->pending.M == 0) { + // NOTE. Here, we can also emit incoming right away. It will make the + // encoder 1.5x faster, at a cost of ~10% lower compression ratio: + // EMIT_MATCH(incoming); + // state->pending = NO_MATCH; + + // No pending match, emit nothing, keep incoming + state->pending = incoming; + } else { + // Here we have both incoming and pending + if (state->pending.m_end <= incoming.m_begin) { + // No overlap: emit pending, keep incoming + EMIT_MATCH(state->pending); + state->pending = incoming; + } else { + // If pending is better, emit pending and discard incoming. + // Otherwise, emit incoming and discard pending. + if (incoming.K > state->pending.K) + state->pending = incoming; + EMIT_MATCH(state->pending); + state->pending = NO_MATCH; + } + } + + after_emit: + + // We commit state changes only after we tried to emit instructions, so we + // can restart in the same state in case dst was full and we quit the loop. + state->table[h] = updated_e; + + } // i loop + + // Do not emit pending match here. We do it only at the end of stream. +} + +// =============================================================== +// API entry points + +size_t lzvn_encode_scratch_size(void) { return LZVN_ENCODE_WORK_SIZE; } + +static size_t lzvn_encode_partial(void *__restrict dst, size_t dst_size, + const void *__restrict src, size_t src_size, + size_t *src_used, void *__restrict work) { + // Min size checks to avoid accessing memory outside buffers. + if (dst_size < LZVN_ENCODE_MIN_DST_SIZE) { + *src_used = 0; + return 0; + } + // Max input size check (limit to offsets on uint32_t). + if (src_size > LZVN_ENCODE_MAX_SRC_SIZE) { + src_size = LZVN_ENCODE_MAX_SRC_SIZE; + } + + // Setup encoder state + lzvn_encoder_state state; + memset(&state, 0, sizeof(state)); + + state.src = src; + state.src_begin = 0; + state.src_end = (lzvn_offset)src_size; + state.src_literal = 0; + state.src_current = 0; + state.dst = dst; + state.dst_begin = dst; + state.dst_end = (unsigned char *)dst + dst_size - 8; // reserve 8 bytes for end-of-stream + state.table = work; + + // Do not encode if the input buffer is too small. We'll emit a literal instead. + if (src_size >= LZVN_ENCODE_MIN_SRC_SIZE) { + + state.src_current_end = (lzvn_offset)src_size - LZVN_ENCODE_MIN_MARGIN; + lzvn_init_table(&state); + lzvn_encode(&state); + + } + + // No need to test the return value: src_literal will not be updated on failure, + // and we will fail later. + lzvn_emit_literal(&state, state.src_end - state.src_literal); + + // Restore original size, so end-of-stream always succeeds, and emit it + state.dst_end = (unsigned char *)dst + dst_size; + lzvn_emit_end_of_stream(&state); + + *src_used = state.src_literal; + return (size_t)(state.dst - state.dst_begin); +} + +size_t lzvn_encode_buffer(void *__restrict dst, size_t dst_size, + const void *__restrict src, size_t src_size, + void *__restrict work) { + size_t src_used = 0; + size_t dst_used = + lzvn_encode_partial(dst, dst_size, src, src_size, &src_used, work); + if (src_used != src_size) + return 0; // could not encode entire input stream = fail + return dst_used; // return encoded size +} diff --git a/view/kernelcache/core/transformers/liblzfse/lzvn_encode_base.h b/view/kernelcache/core/transformers/liblzfse/lzvn_encode_base.h new file mode 100644 index 00000000..1b93d0e8 --- /dev/null +++ b/view/kernelcache/core/transformers/liblzfse/lzvn_encode_base.h @@ -0,0 +1,116 @@ +/* +Copyright (c) 2015-2016, Apple Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder(s) nor the names of any contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +// LZVN low-level encoder + +#ifndef LZVN_ENCODE_BASE_H +#define LZVN_ENCODE_BASE_H + +#include "lzfse_internal.h" + +// =============================================================== +// Types and Constants + +#define LZVN_ENCODE_HASH_BITS \ + 14 // number of bits returned by the hash function [10, 16] +#define LZVN_ENCODE_OFFSETS_PER_HASH \ + 4 // stored offsets stack for each hash value, MUST be 4 +#define LZVN_ENCODE_HASH_VALUES \ + (1 << LZVN_ENCODE_HASH_BITS) // number of entries in hash table +#define LZVN_ENCODE_MAX_DISTANCE \ + 0xffff // max match distance we can represent with LZVN encoding, MUST be + // 0xFFFF +#define LZVN_ENCODE_MIN_MARGIN \ + 8 // min number of bytes required between current and end during encoding, + // MUST be >= 8 +#define LZVN_ENCODE_MAX_LITERAL_BACKLOG \ + 400 // if the number of pending literals exceeds this size, emit a long + // literal, MUST be >= 271 + +/*! @abstract Type of table entry. */ +typedef struct { + int32_t indices[4]; // signed indices in source buffer + uint32_t values[4]; // corresponding 32-bit values +} lzvn_encode_entry_type; + +// Work size +#define LZVN_ENCODE_WORK_SIZE \ + (LZVN_ENCODE_HASH_VALUES * sizeof(lzvn_encode_entry_type)) + +/*! @abstract Match */ +typedef struct { + lzvn_offset m_begin; // beginning of match, current position + lzvn_offset m_end; // end of match, this is where the next literal would begin + // if we emit the entire match + lzvn_offset M; // match length M: m_end - m_begin + lzvn_offset D; // match distance D + lzvn_offset K; // match gain: M - distance storage (L not included) +} lzvn_match_info; + +// =============================================================== +// Internal encoder state + +/*! @abstract Base encoder state and I/O. */ +typedef struct { + + // Encoder I/O + + // Source buffer + const unsigned char *src; + // Valid range in source buffer: we can access src[i] for src_begin <= i < + // src_end. src_begin may be negative. + lzvn_offset src_begin; + lzvn_offset src_end; + // Next byte to process in source buffer + lzvn_offset src_current; + // Next byte after the last byte to process in source buffer. We MUST have: + // src_current_end + 8 <= src_end. + lzvn_offset src_current_end; + // Next byte to encode in source buffer, may be before or after src_current. + lzvn_offset src_literal; + + // Next byte to write in destination buffer + unsigned char *dst; + // Valid range in destination buffer: [dst_begin, dst_end - 1] + unsigned char *dst_begin; + unsigned char *dst_end; + + // Encoder state + + // Pending match + lzvn_match_info pending; + + // Distance for last emitted match, or 0 + lzvn_offset d_prev; + + // Hash table used to find matches. Stores LZVN_ENCODE_OFFSETS_PER_HASH 32-bit + // signed indices in the source buffer, and the corresponding 4-byte values. + // The number of entries in the table is LZVN_ENCODE_HASH_VALUES. + lzvn_encode_entry_type *table; + +} lzvn_encoder_state; + +/*! @abstract Encode source to destination. + * Update \p state. + * The call ensures \c src_literal is never left too far behind \c src_current. */ +void lzvn_encode(lzvn_encoder_state *state); + +#endif // LZVN_ENCODE_BASE_H diff --git a/view/kernelcache/ui/CMakeLists.txt b/view/kernelcache/ui/CMakeLists.txt new file mode 100644 index 00000000..369dc603 --- /dev/null +++ b/view/kernelcache/ui/CMakeLists.txt @@ -0,0 +1,95 @@ +cmake_minimum_required(VERSION 3.9 FATAL_ERROR) + +project(kernelcacheui) + +set(CMAKE_AUTOMOC ON) +set(CMAKE_AUTORCC ON) +find_package(Qt6 COMPONENTS Core Gui Widgets REQUIRED) + +file(GLOB SOURCES *.cpp *.h) +list(FILTER SOURCES EXCLUDE REGEX moc_.*) +list(FILTER SOURCES EXCLUDE REGEX qrc_.*) + +add_library(kernelcacheui SHARED ${SOURCES}) + +set(COMPILE_DEFS "") + +if (HARD_FAIL_MODE) + set(COMPILE_DEFS "${COMPILE_DEFS} ABORT_FAILURES;") +endif() + +if (BN_REF_COUNT_DEBUG) + set(COMPILE_DEFS "${COMPILE_DEFS} BN_REF_COUNT_DEBUG;") +endif() + +if (SLIDEINFO_DEBUG_TAGS) + set(COMPILE_DEFS "${COMPILE_DEFS} SLIDEINFO_DEBUG_TAGS;") +endif() + +if (METADATA_VERSION) + set(COMPILE_DEFS "${COMPILE_DEFS} METADATA_VERSION=${METADATA_VERSION};") +else() + message(FATAL_ERROR "No metadata version provided. Fatal.") +endif() + +if (KC_VIEW_NAME) + set(COMPILE_DEFS "${COMPILE_DEFS} KC_VIEW_NAME=\"${KC_VIEW_NAME}\";") +else() + message(FATAL_ERROR "No view name provided. Fatal.") +endif() + +target_compile_definitions(kernelcacheui PRIVATE ${COMPILE_DEFS}) + + +if(BN_INTERNAL_BUILD) + set_target_properties(kernelcacheui PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${BN_CORE_PLUGIN_DIR} + RUNTIME_OUTPUT_DIRECTORY ${BN_CORE_PLUGIN_DIR}) +else() + set_target_properties(kernelcacheui PROPERTIES + LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/out/plugins + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/out/plugins + ) +endif() + +set_target_properties(kernelcacheui PROPERTIES + CXX_STANDARD 17 + CXX_STANDARD_REQUIRED ON + CXX_VISIBILITY_PRESET hidden + VISIBILITY_INLINES_HIDDEN ON + POSITION_INDEPENDENT_CODE ON + ) + +function(get_recursive_include_dirs target result) + # Initialize an empty list to store include directories + set(include_dirs "") + + # Get the include directories of the current target + get_target_property(current_target_includes ${target} INTERFACE_INCLUDE_DIRECTORIES) + if(current_target_includes) + list(APPEND include_dirs ${current_target_includes}) + endif() + + # Get the libraries that this target links to + get_target_property(linked_libraries ${target} INTERFACE_LINK_LIBRARIES) + if(linked_libraries) + foreach(linked_library IN LISTS linked_libraries) + # Skip plain library names (non-target libraries) + if(TARGET ${linked_library}) + # Recursively get include directories from linked libraries + get_recursive_include_dirs(${linked_library} linked_library_includes) + list(APPEND include_dirs ${linked_library_includes}) + endif() + endforeach() + endif() + + # Set the result to the collected include directories + set(${result} ${include_dirs} PARENT_SCOPE) +endfunction() + +get_recursive_include_dirs(kernelcacheapi INCLUDES) + +target_include_directories(kernelcacheui PRIVATE ${INCLUDES}) + +target_link_libraries(kernelcacheui kernelcacheapi kernelcache binaryninjaui Qt6::Core Qt6::Gui Qt6::Widgets) + diff --git a/view/kernelcache/ui/KernelCacheUINotifications.cpp b/view/kernelcache/ui/KernelCacheUINotifications.cpp new file mode 100644 index 00000000..6bd8fdf0 --- /dev/null +++ b/view/kernelcache/ui/KernelCacheUINotifications.cpp @@ -0,0 +1,92 @@ +// +// Created by kat on 5/8/23. +// + +#include "KernelCacheUINotifications.h" +#include <QLayout> +#include <kernelcacheapi.h> +#include "ui/sidebar.h" +#include "ui/linearview.h" +#include "ui/viewframe.h" +#include "progresstask.h" + +UINotifications* UINotifications::m_instance = nullptr; + +void UINotifications::init() +{ + m_instance = new UINotifications; + UIContext::registerNotification(m_instance); +} + + +void UINotifications::OnViewChange(UIContext* context, ViewFrame* frame, const QString& type) +{ + if (!frame) + return; + + // FIXME there is a bv func for this + static std::function<bool(Ref<BinaryView>, uint64_t)> isAddrMapped = [](Ref<BinaryView> view, uint64_t addr) { + if (view && view->GetTypeName() == KC_VIEW_NAME) + { + for (const auto& seg : view->GetSegments()) + { + if (seg->GetStart() <= addr && seg->GetEnd() > addr) + return true; + } + } + return false; + }; + + auto view = frame->getCurrentBinaryView(); + if (view && view->GetTypeName() == KC_VIEW_NAME) + { + if (auto viewInt = frame->getCurrentViewInterface()) + { + auto ah = viewInt->actionHandler(); + if (!ah->isBoundAction("KC Load IMGHERE")) + { + ah->bindAction("KC Load IMGHERE", + UIAction( + [](const UIActionContext& ctx) { + Ref<BinaryView> view = ctx.binaryView; + Ref<KernelCacheAPI::KernelCache> cache = new KernelCacheAPI::KernelCache(view); + uint64_t addr = ctx.token.token.value; + if (addr) + { + BackgroundThread::create(ctx.context->mainWindow())->thenBackground( + [cache=cache, addr=addr]() { + cache->LoadImageContainingAddress(addr); + })->start(); + } + }, + [](const UIActionContext& ctx) { + Ref<KernelCacheAPI::KernelCache> cache = new KernelCacheAPI::KernelCache(ctx.binaryView); + uint64_t addr = ctx.token.token.value; + if (isAddrMapped(ctx.binaryView, addr)) + return false; + return addr && cache->GetImageNameForAddress(addr) != ""; // bool + })); + ah->setActionDisplayName("KC Load IMGHERE", [](const UIActionContext& ctx) { + Ref<KernelCacheAPI::KernelCache> cache = new KernelCacheAPI::KernelCache(ctx.binaryView); + uint64_t addr = ctx.token.token.value; + if (addr) + return QString("Load ") + cache->GetImageNameForAddress(addr).c_str(); + return QString("Error"); + }); + if (auto linearView = qobject_cast<LinearView*>(viewInt->widget())) + { + linearView->contextMenu().addAction("KC Load IMGHERE", KC_VIEW_NAME); + linearView->contextMenu().setGroupOrdering(KC_VIEW_NAME, 0); + } + } + } + } +} +void UINotifications::OnAfterOpenFile(UIContext* context, FileContext* file, ViewFrame* frame) +{ + if (frame->getCurrentBinaryView()) + { + // Register BD notifications. We dont use them right now. + } + UIContextNotification::OnAfterOpenFile(context, file, frame); +} diff --git a/view/kernelcache/ui/KernelCacheUINotifications.h b/view/kernelcache/ui/KernelCacheUINotifications.h new file mode 100644 index 00000000..9ea4b50d --- /dev/null +++ b/view/kernelcache/ui/KernelCacheUINotifications.h @@ -0,0 +1,23 @@ +// +// Created by kat on 5/8/23. +// +#include "ui/uicontext.h" + +#ifndef SHAREDCACHE_NOTIFICATIONS_H +#define SHAREDCACHE_NOTIFICATIONS_H + +class UINotifications : public UIContextNotification { + static UINotifications* m_instance; + + std::vector<size_t> m_sessionsAlreadyDisplayedPickerFor; + +public: + virtual void OnViewChange(UIContext *context, ViewFrame *frame, const QString &type) override; + // bool OnAfterOpenDatabase(UIContext* context, FileMetadataRef metadata, BinaryViewRef data) override; + void OnAfterOpenFile(UIContext* context, FileContext* file, ViewFrame* frame) override; + + static void init(); +}; + + +#endif //SHAREDCACHE_NOTIFICATIONS_H diff --git a/view/kernelcache/ui/Plugin.cpp b/view/kernelcache/ui/Plugin.cpp new file mode 100644 index 00000000..cfa92fe4 --- /dev/null +++ b/view/kernelcache/ui/Plugin.cpp @@ -0,0 +1,22 @@ +// +// Created by kat on 8/6/24. +// +#include <binaryninjaapi.h> +#include "KernelCacheUINotifications.h" +#include "kctriage.h" + +extern "C" +{ + BN_DECLARE_CORE_ABI_VERSION + BN_DECLARE_UI_ABI_VERSION + + BINARYNINJAPLUGIN bool UIPluginInit() + { + UINotifications::init(); + UIAction::registerAction("KC Load IMGHERE"); + + KCTriageViewType::Register(); + + return true; + } +}
\ No newline at end of file diff --git a/view/kernelcache/ui/kctriage.cpp b/view/kernelcache/ui/kctriage.cpp new file mode 100644 index 00000000..5f0fa9ce --- /dev/null +++ b/view/kernelcache/ui/kctriage.cpp @@ -0,0 +1,556 @@ +// +// Created by kat on 8/15/24. +// + +#include "kctriage.h" +#include "ui/fontsettings.h" +#include <QPainter> +#include <QTextBrowser> +#include "tabwidget.h" +#include "globalarea.h" +#include "progresstask.h" + +#include <cmath> +#include <QMessageBox> + + +#define QSETTINGS_KEY_SELECTED_TAB "KCTriage-SelectedTab" +#define QSETTINGS_KEY_TAB_LAYOUT "KCTriage-TabLayout" +#define QSETTINGS_KEY_IMAGELOAD_TAB_LAYOUT "KCTriage-ImageLoadTabLayout" +#define QSETTINGS_KEY_ALPHA_POPUP_SEEN "KCTriage-AlphaPopupSeenV2" + + +SymbolTableModel::SymbolTableModel(SymbolTableView* parent) + : QAbstractTableModel(parent), m_parent(parent) { +} + +int SymbolTableModel::rowCount(const QModelIndex& parent) const { + Q_UNUSED(parent); + return static_cast<int>(m_symbols.size()); +} + +int SymbolTableModel::columnCount(const QModelIndex& parent) const { + Q_UNUSED(parent); + // We have 3 columns: Address, Name, and Image + return 3; +} + +QVariant SymbolTableModel::data(const QModelIndex& index, int role) const { + if (!index.isValid() || role != Qt::DisplayRole) { + return QVariant(); + } + + const KernelCacheAPI::KCSymbol& symbol = m_symbols.at(index.row()); + + switch (index.column()) { + case 0: // Address column + return QString("0x%1").arg(symbol.address, 0, 16); // Display address as hexadecimal + case 1: // Name column + return QString::fromStdString(symbol.name); + case 2: // Image column + return QString::fromStdString(symbol.image); + default: + return QVariant(); + } +} + +QVariant SymbolTableModel::headerData(int section, Qt::Orientation orientation, int role) const { + if (role != Qt::DisplayRole || orientation != Qt::Horizontal) { + return QVariant(); + } + + switch (section) { + case 0: + return QString("Address"); + case 1: + return QString("Name"); + case 2: + return QString("Image"); + default: + return QVariant(); + } +} + +void SymbolTableModel::updateSymbols() { + m_symbols = m_parent->m_symbols; + setFilter(m_filter); +} + +const KernelCacheAPI::KCSymbol& SymbolTableModel::symbolAt(int row) const { + return m_symbols.at(row); +} + + +void SymbolTableModel::setFilter(std::string text) +{ + beginResetModel(); + + m_filter = text; + m_symbols.clear(); + + if (m_filter.empty()) + { + m_symbols = m_parent->m_symbols; + } + else + { + m_symbols.reserve(m_parent->m_symbols.size()); + for (const auto& symbol : m_parent->m_symbols) + { + if (symbol.name.find(m_filter) != std::string::npos) + { + m_symbols.push_back(symbol); + } + } + m_symbols.shrink_to_fit(); + } + + endResetModel(); +} + + +SymbolTableView::SymbolTableView(QWidget* parent, Ref<KernelCacheAPI::KernelCache> cache) + : m_model(new SymbolTableModel(this)) { + + // Set up the filter model + setModel(m_model); + + // Configure view settings + horizontalHeader()->setSectionResizeMode(QHeaderView::Stretch); + setSelectionBehavior(QAbstractItemView::SelectRows); + setSelectionMode(QAbstractItemView::SingleSelection); + + BackgroundThread::create(this)->thenBackground([this, cache=cache](){ + // LogInfo("Symbol Search: Loading symbols..."); + m_symbols = cache->LoadAllSymbolsAndWait(); + // LogInfo("Symbol Search: Loaded 0x%zx symbols", m_symbols.size()); + })->thenMainThread([this](){ + m_model->updateSymbols(); + })->start(); +} + +SymbolTableView::~SymbolTableView() { + delete m_model; +} + +void SymbolTableView::setFilter(const std::string& filter) { + m_model->setFilter(filter); +} + + +KCTriageView::KCTriageView(QWidget* parent, BinaryViewRef data) : QWidget(parent), View(), m_data(data), m_cache(new KernelCacheAPI::KernelCache(data)) +{ + setBinaryDataNavigable(true); + setupView(this); + + m_triageCollection = new DockableTabCollection(); + m_triageTabs = new SplitTabWidget(m_triageCollection); + + auto triageTabStyle = new GlobalAreaTabStyle(); + m_triageTabs->setTabStyle(triageTabStyle); + + QSplitter* containerWidget = new QSplitter; + containerWidget->setOrientation(Qt::Vertical); + + QWidget* defaultWidget = nullptr; + + m_bottomRegionCollection = new DockableTabCollection(); + m_bottomRegionTabs = new SplitTabWidget(m_bottomRegionCollection); + m_bottomRegionTabs->setTabStyle(new GlobalAreaTabStyle()); + + auto loadImageTable = new FilterableTableView; + { + auto loadImageModel = new QStandardItemModel(0, 2, loadImageTable); + { + loadImageModel->setHorizontalHeaderLabels({"Name", "VM Address"}); + BackgroundThread::create(loadImageTable)->thenBackground([this, loadImageModel](QVariant var) + { + QVariantList rows; + + auto images = m_cache->GetImages(); + + auto newHeaders = std::make_shared<std::vector<KernelCacheAPI::KernelCacheMachOHeader>>(); + newHeaders->reserve(images.size()); + + for (const auto& img : images) + { + if (auto header = m_cache->GetMachOHeaderForImage(img.name); header) + { + newHeaders->push_back(*header); + rows.push_back(QList<QVariant>{ + QString::fromStdString(img.name), + QString("0x%1").arg(header->textBase, 0, 16) + }); + } + } + + { + std::unique_lock<std::mutex> lock(m_headersMutex); + m_headers.swap(newHeaders); + } + + return QVariant(rows); + })->thenMainThread([this, loadImageModel, loadImageTable](QVariant var){ + QVariantList rows = var.toList(); + + if (loadImageModel->rowCount() > 0) + { + loadImageModel->removeRows(0, loadImageModel->rowCount()); + } + + for (const QVariant &rowVariant : rows) { + QVariantList row = rowVariant.toList(); + + QList<QStandardItem*> items; + for (const QVariant &cellValue : row) { + items.append(new QStandardItem(cellValue.toString())); + } + + loadImageModel->appendRow(items); + loadImageTable->resizeColumnsToContents(); + } + })->start(); + } // loadImageModel + + auto loadImageButton = new CustomStyleFlatPushButton(); + { + connect(loadImageButton, &QPushButton::clicked, + [this, loadImageTable](bool) { + auto selected = loadImageTable->selectionModel()->selectedRows(); + if (selected.size() == 0) + { + return; + } + + for (const auto& selection : selected) + { + auto name = selection.data().toString().toStdString(); + WorkerPriorityEnqueue([this, name]() { m_cache->LoadImageWithInstallName(name); }); + } + }); + loadImageButton->setText("Load"); + + loadImageButton->setSizePolicy(QSizePolicy::Fixed, QSizePolicy::Fixed); + loadImageButton->setMinimumWidth(100); + loadImageButton->setMinimumHeight(30); + + } // loadImageButton + loadImageTable->setEditTriggers(QAbstractItemView::NoEditTriggers); + + auto loadImageFilterEdit = new FilterEdit(loadImageTable); + { + connect(loadImageFilterEdit, &FilterEdit::textChanged, [loadImageTable](const QString& filter) { + loadImageTable->setFilter(filter.toStdString()); + }); + } // loadImageFilterEdit + + connect(loadImageTable, &FilterableTableView::activated, this, [=](const QModelIndex& index) + { + auto selected = loadImageTable->selectionModel()->selectedRows(); + if (selected.size() == 0) + { + return; + } + + for (const auto& selection : selected) + { + auto name = selection.data().toString().toStdString(); + WorkerPriorityEnqueue([this, name]() { m_cache->LoadImageWithInstallName(name); }); + } + }); + connect(loadImageTable, &FilterableTableView::doubleClicked, this, [=](const QModelIndex& index) + { + auto selected = loadImageTable->selectionModel()->selectedRows(); + if (selected.size() == 0) + { + return; + } + + for (const auto& selection : selected) + { + auto name = selection.data().toString().toStdString(); + WorkerPriorityEnqueue([this, name]() { m_cache->LoadImageWithInstallName(name); }); + } + }); + + auto loadImageLayout = new QVBoxLayout; + loadImageLayout->addWidget(loadImageFilterEdit); + loadImageLayout->addWidget(loadImageTable); + loadImageLayout->addWidget(loadImageButton); + + auto loadImageWidget = new QWidget; + loadImageWidget->setLayout(loadImageLayout); + + m_bottomRegionTabs->addTab(loadImageWidget, "Load an Image"); + + loadImageTable->setModel(loadImageModel); + + loadImageTable->horizontalHeader()->setSectionResizeMode(0, QHeaderView::Stretch); + loadImageTable->horizontalHeader()->setSectionResizeMode(1, QHeaderView::ResizeToContents); + + loadImageTable->setSelectionBehavior(QAbstractItemView::SelectRows); + + m_triageTabs->addTab(loadImageWidget, "Images"); + defaultWidget = loadImageWidget; + m_triageTabs->setCanCloseTab(loadImageWidget, false); + } // loadImageTable + + auto symbolSearch = new SymbolTableView(this, m_cache); + { + auto symbolFilterEdit = new FilterEdit(symbolSearch); + { + connect(symbolFilterEdit, &FilterEdit::textChanged, [symbolSearch](const QString& filter) { + symbolSearch->setFilter(filter.toStdString()); + }); + } + + auto symbolLayout = new QVBoxLayout; + symbolLayout->addWidget(symbolFilterEdit); + symbolLayout->addWidget(symbolSearch); + + auto symbolWidget = new QWidget; + symbolWidget->setLayout(symbolLayout); + + symbolSearch->horizontalHeader()->setSectionResizeMode(0, QHeaderView::ResizeToContents); // Address + symbolSearch->horizontalHeader()->setSectionResizeMode(1, QHeaderView::Stretch); // Name + symbolSearch->horizontalHeader()->setSectionResizeMode(2, QHeaderView::Stretch); // Image + + symbolSearch->setSelectionBehavior(QAbstractItemView::SelectRows); + symbolSearch->setSelectionMode(QAbstractItemView::SingleSelection); + + connect(symbolSearch, &SymbolTableView::activated, this, [=](const QModelIndex& index) + { + auto symbol = symbolSearch->getSymbolAtRow(index.row()); + auto dialog = new QMessageBox(this); + dialog->setText("Load " + QString::fromStdString(symbol.image) + "?"); + dialog->setStandardButtons(QMessageBox::Yes | QMessageBox::No); + + connect(dialog, &QMessageBox::buttonClicked, this, [=](QAbstractButton* button) + { + if (button == dialog->button(QMessageBox::Yes)) + { + WorkerPriorityEnqueue([this, symbol]() + { + m_cache->LoadImageWithInstallName(symbol.image); + }); + } + }); + dialog->exec(); + }); + + m_triageTabs->addTab(symbolWidget, "Symbol Search"); + m_triageTabs->setCanCloseTab(symbolWidget, false); + } // symbolSearch + + auto loadedRegions = new QTreeView; + { + auto loadedRegionsModel = new QStandardItemModel(0, 3, loadedRegions); + loadedRegionsModel->setHorizontalHeaderLabels({"VM Address", "Size", "Pretty Name"}); + + auto loadedRegionsLayout = new QVBoxLayout; + loadedRegionsLayout->addWidget(loadedRegions); + + auto loadedRegionsWidget = new QWidget; + loadedRegionsWidget->setLayout(loadedRegionsLayout); + + loadedRegions->setModel(loadedRegionsModel); + + loadedRegions->header()->setSectionResizeMode(QHeaderView::Stretch); + + loadedRegions->setSelectionBehavior(QAbstractItemView::SelectRows); + loadedRegions->setSelectionMode(QAbstractItemView::SingleSelection); + + connect(loadedRegions, &QTreeView::doubleClicked, this, [=](const QModelIndex& index) + { + auto addr = loadedRegionsModel->item(index.row(), 0)->text().toULongLong(nullptr, 16); + }); + + connect(loadedRegions, &QTreeView::activated, this, [=](const QModelIndex& index) + { + auto addr = loadedRegionsModel->item(index.row(), 0)->text().toULongLong(nullptr, 16); + }); + + // m_triageTabs->addTab(loadedRegionsWidget, "Loaded Regions"); + } // loadedRegions + + + { // Doc tabs + + QTextBrowser *mainDocBrowser = new QTextBrowser(this); + { + mainDocBrowser->setOpenExternalLinks(true); + auto alphaHtml = + R"( +<h1>KernelCache Parser</h1> + +<p> This parser has support for MH_FILESET (>= macOS 11, >= iOS 14) kernels. </p> + +<h2> Getting the latest version of the plugin </h2> +<p> We frequently release "dev" builds which will contain the latest version of the KernelCache plugin (and many other things). </p> +<p> You can find instructions on how to install these builds <a href="https://docs.binary.ninja/guide/index.html#development-branch">here</a>. </p> + +<h3> Reading / building the source </h3> +<p> Like most of our platforms, architectures, debug information parsing, and the entire API and documentation, this plugin is open source. </p> +<p> You can read the source and find instructions for building it <a href="https://github.com/Vector35/binaryninja-api/tree/dev/view/kernelcache">here</a>. </p> +<p> Contributions are always welcome! </p> +)"; + mainDocBrowser->setHtml(alphaHtml); + + m_triageTabs->addTab(mainDocBrowser, "KernelCache Parser"); + m_triageTabs->setCanCloseTab(mainDocBrowser, false); + + } + } + + containerWidget->addWidget(m_bottomRegionTabs); + + m_layout = new QVBoxLayout(this); + m_layout->addWidget(m_triageTabs); + setLayout(m_layout); + + m_triageTabs->selectWidget(defaultWidget); +} + + +QFont KCTriageView::getFont() +{ + return getMonospaceFont(this); +} + + +BinaryViewRef KCTriageView::getData() +{ + return m_data; +} + + +bool KCTriageView::navigate(uint64_t offset) +{ + return false; +} + + +uint64_t KCTriageView::getCurrentOffset() +{ + return 0; +} + + +CollapsibleSection::CollapsibleSection(QWidget* parent) + : QWidget(parent) +{ + auto layout = new QVBoxLayout(this); + { + layout->setContentsMargins(0, 0, 0, 0); + + auto hLayout = new QHBoxLayout; + { + hLayout->setContentsMargins(0, 0, 0, 0); + + m_titleLabel = new QLabel; + m_titleLabel->setStyleSheet("font-weight: bold; font-size: 16px;"); + hLayout->addWidget(m_titleLabel, 1); + + m_subtitleRightLabel = new QLabel; + m_subtitleRightLabel->setStyleSheet("font-size: 12px;"); + hLayout->addWidget(m_subtitleRightLabel); + + m_collapseButton = new CustomStyleFlatPushButton; + m_collapseButton->setFlat(true); + m_collapseButton->setCheckable(true); + } + + layout->addLayout(hLayout); + } + + m_contentWidgetContainer = new QWidget; + { + layout->addWidget(m_contentWidgetContainer); + new QVBoxLayout(m_contentWidgetContainer); + } + +} + + +void CollapsibleSection::setTitle(const QString& title) +{ + m_titleLabel->setText(title); +} + + +void CollapsibleSection::setSubtitleRight(const QString& subtitle) +{ + m_subtitleRightLabel->setVisible(subtitle != ""); + m_subtitleRightLabel->setText(subtitle); +} + + +void CollapsibleSection::setContentWidget(QWidget* contentWidget) +{ + m_contentWidget = contentWidget; + m_contentWidgetContainer->layout()->addWidget(contentWidget); +} + + +QSize CollapsibleSection::sizeHint() const +{ + return QWidget::sizeHint(); +} + + +void CollapsibleSection::setCollapsed(bool collapsed, bool animated) +{ + if (collapsed == m_collapsed) + { + return; + } + + m_collapsed = collapsed; + + if (m_collapsed) + { + m_contentWidget->hide(); + } + else + { + m_contentWidget->show(); + } + + if (animated) + { + m_onContentAddedAnimation->start(); + } +} + + +KCTriageViewType::KCTriageViewType() + : ViewType("KCTriage", "Kernel Cache Triage") +{ + +} + + +int KCTriageViewType::getPriority(BinaryViewRef data, const QString& filename) +{ + if (data->GetTypeName() == KC_VIEW_NAME) + { + return 100; + } + return 0; +} + + +QWidget* KCTriageViewType::create(BinaryViewRef data, ViewFrame* viewFrame) +{ + if (data->GetTypeName() != KC_VIEW_NAME) + { + return nullptr; + } + return new KCTriageView(viewFrame, data); +} + + +void KCTriageViewType::Register() +{ + ViewType::registerViewType(new KCTriageViewType()); +} diff --git a/view/kernelcache/ui/kctriage.h b/view/kernelcache/ui/kctriage.h new file mode 100644 index 00000000..a6b7f7ac --- /dev/null +++ b/view/kernelcache/ui/kctriage.h @@ -0,0 +1,243 @@ +// +// Created by kat on 8/15/24. +// + +#include <kernelcacheapi.h> +#include <binaryninjaapi.h> +#include "uitypes.h" +#include "viewframe.h" +#include "animation.h" +#include "uicontext.h" + +#include <QTableView> +#include <QStandardItemModel> +#include <QSortFilterProxyModel> +#include <QHeaderView> +#include "filter.h" + +#ifndef BINARYNINJA_KCTRIAGE_H +#define BINARYNINJA_KCTRIAGE_H + +class CollapsibleSection : public QWidget +{ + Q_OBJECT + + QLabel* m_titleLabel; + QLabel* m_subtitleRightLabel; + QPushButton* m_collapseButton; + + bool m_collapsed = true; + + Animation* m_onContentAddedAnimation; + + QWidget* m_contentWidgetContainer; + QWidget* m_contentWidget; + +protected: + QSize sizeHint() const override; + +public: + CollapsibleSection(QWidget* parent); + void setTitle(const QString& title); + void setSubtitleRight(const QString& subtitle); + + void setContentWidget(QWidget* contentWidget); + + void setCollapsed(bool collapsed, bool animated = true); + bool isCollapsed() const { return m_collapsed; } +}; + + +class FilterableTableView : public QTableView, public FilterTarget { + Q_OBJECT + + bool m_filterByHiding; + +public: + FilterableTableView(QWidget* parent = nullptr, bool filterByHiding = true) + : QTableView(parent), m_filterByHiding(filterByHiding) { + viewport()->installEventFilter(this); + } + + ~FilterableTableView() override {} + + void setFilter(const std::string& filter) override { + if (!m_filterByHiding) + { + emit filterTextChanged(QString::fromStdString(filter)); + return; + } + QString qFilter = QString::fromStdString(filter); + for (int row = 0; row < model()->rowCount(); ++row) { + bool match = false; + for (int col = 0; col < model()->columnCount(); ++col) { + QModelIndex index = model()->index(row, col); + QString data = model()->data(index).toString(); + if (data.contains(qFilter, Qt::CaseInsensitive)) { + match = true; + break; + } + } + setRowHidden(row, !match); + } + } + + void scrollToFirstItem() override { + if (model()->rowCount() > 0) { + scrollTo(model()->index(0, 0)); + } + } + + void scrollToCurrentItem() override { + QModelIndex currentIndex = selectionModel()->currentIndex(); + if (currentIndex.isValid()) { + scrollTo(currentIndex); + } + } + + void selectFirstItem() override { + if (model()->rowCount() > 0) { + QModelIndex firstIndex = model()->index(0, 0); + selectionModel()->select(firstIndex, QItemSelectionModel::ClearAndSelect); + } + } + + void activateFirstItem() override { + if (model()->rowCount() > 0) { + QModelIndex firstIndex = model()->index(0, 0); + setCurrentIndex(firstIndex); + emit activated(firstIndex); + } + } + + bool eventFilter(QObject* obj, QEvent* event) override { + if (event->type() == QEvent::KeyPress) { + QKeyEvent* keyEvent = static_cast<QKeyEvent*>(event); + if (keyEvent->key() == Qt::Key_Escape) { + clearSelection(); + return true; + } + if (keyEvent->key() == Qt::Key_Enter || keyEvent->key() == Qt::Key_Return) { + emit activated(currentIndex()); + return true; + } + } + return QTableView::eventFilter(obj, event); + } + +signals: + void filterTextChanged(const QString& text); +}; + +class SymbolTableView; + +class SymbolTableModel : public QAbstractTableModel { + Q_OBJECT + + SymbolTableView* m_parent; + std::string m_filter; + std::vector<KernelCacheAPI::KCSymbol> m_symbols; + +public: + explicit SymbolTableModel(SymbolTableView* parent); + + int rowCount(const QModelIndex& parent = QModelIndex()) const override; + int columnCount(const QModelIndex& parent = QModelIndex()) const override; + QVariant data(const QModelIndex& index, int role = Qt::DisplayRole) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role = Qt::DisplayRole) const override; + + void updateSymbols(); + + void setFilter(std::string text); + + const KernelCacheAPI::KCSymbol& symbolAt(int row) const; +}; + + +class SymbolTableView : public QTableView, public FilterTarget +{ + Q_OBJECT + friend class SymbolTableModel; + + std::vector<KernelCacheAPI::KCSymbol> m_symbols; + + SymbolTableModel* m_model; + +public: + SymbolTableView(QWidget* parent, Ref<KernelCacheAPI::KernelCache> cache); + virtual ~SymbolTableView() override; + + void scrollToFirstItem() override { + if (model()->rowCount() > 0) { + scrollTo(model()->index(0, 0)); + } + } + + void scrollToCurrentItem() override { + QModelIndex currentIndex = selectionModel()->currentIndex(); + if (currentIndex.isValid()) { + scrollTo(currentIndex); + } + } + + void selectFirstItem() override { + if (model()->rowCount() > 0) { + QModelIndex firstIndex = model()->index(0, 0); + selectionModel()->select(firstIndex, QItemSelectionModel::ClearAndSelect); + } + } + + void activateFirstItem() override { + if (model()->rowCount() > 0) { + QModelIndex firstIndex = model()->index(0, 0); + setCurrentIndex(firstIndex); + emit activated(firstIndex); + } + } + + KernelCacheAPI::KCSymbol getSymbolAtRow(int row) const + { + return m_model->symbolAt(row); + } + + void setFilter(const std::string& filter) override; +}; + + +class KCTriageView : public QWidget, public View +{ + BinaryViewRef m_data; + QVBoxLayout* m_layout; + Ref<KernelCacheAPI::KernelCache> m_cache; + + std::mutex m_headersMutex; + std::shared_ptr<std::vector<KernelCacheAPI::KernelCacheMachOHeader>> m_headers; + + SplitTabWidget* m_triageTabs; + DockableTabCollection* m_triageCollection; + + SplitTabWidget* m_bottomRegionTabs; + DockableTabCollection* m_bottomRegionCollection; + + +public: + KCTriageView(QWidget* parent, BinaryViewRef data); + BinaryViewRef getData() override; + void setSelectionOffsets(BNAddressRange range) override {}; + QFont getFont() override; + bool navigate(uint64_t offset) override; + uint64_t getCurrentOffset() override; +}; + + +class KCTriageViewType : public ViewType +{ +public: + KCTriageViewType(); + int getPriority(BinaryViewRef data, const QString& filename) override; + QWidget* create(BinaryViewRef data, ViewFrame* viewFrame) override; + static void Register(); +}; + + +#endif // BINARYNINJA_KCTRIAGE_H |
