From 1ffdd4da176f1990cdf0ff4d0079a3ccb4e51206 Mon Sep 17 00:00:00 2001 From: Mark Rowe Date: Thu, 30 Jan 2025 09:48:06 -0800 Subject: [SharedCache] Rework how file accessors are handled Previously, `MMappedFileAccessor::Open` attempted to impose a fixed limit on the number of file accessors that were live at one time. This was done because the default file descriptor limit on some platforms is relatively low (256 on macOS). Given that recent iOS shared caches can contain 60+ files it is easy to tie up a large percentage of this limit by opening one or two shared caches. This is problematic as if the limit imposed by `MMappedFileAccessor::Open` is reached, the attempt to access the file will block waiting for another file accessor to be closed. This can lead to a deadlock. This commit makes three changes to this strategy: 1. It attempts to raise the file descriptor limit to 1024. Unix systems support both soft and hard file descriptor limits. The soft file descriptor limit is what is enforced, but a process can explicitly raise the limit to any value below the hard limit if it wishes. 2. The fixed limit on open files accessors is changed to a soft limit. `FileAccessorCache` is introduced to manage the caching of file accessors. It provides a basic LRU cache. Whenever a new file is opened, the cache of open accessors is pruned to stay below the target limit (50% of the soft file descriptor limit). This limiting is primarily done to allow files containing dirty pages to be unmapped if they're no longer being used. LRU isn't the optimal strategy for this, but it is simple to implement and understand. Ideally there'd be some access time component to the cache so files that haven't been accessed can be released. 3. File accessors are cached even for sessions corresponding to views that have been closed. The "Open Selection with Options..." context menu hits this case as a view is created and destroyed as part of populating the options dialog, and the real view is then created in the same session. The most significant benefit of this change is that the logic around opening / closing file accessors is simpler and can no longer deadlock. While working on this I noticed that `SharedCache` opened some files via `MMappedFileAccessor::Open` without specifying a post-open operation to apply slide information. Instead, it would explicitly apply the slide information after opening the file. This works fine so long as the file accessor limit is never hit. If it is hit and the accessor is closed, the next time the file is opened it will not have any slide information applied. This would lead to very confusing bugs. --- view/sharedcache/core/DSCView.cpp | 2 - view/sharedcache/core/SharedCache.cpp | 68 ++++++------ view/sharedcache/core/SharedCache.h | 6 +- view/sharedcache/core/VM.cpp | 201 +++++++++++++++++++++------------- view/sharedcache/core/VM.h | 110 +++++++++---------- 5 files changed, 209 insertions(+), 178 deletions(-) (limited to 'view/sharedcache/core') diff --git a/view/sharedcache/core/DSCView.cpp b/view/sharedcache/core/DSCView.cpp index c0b7b349..974e3c4e 100644 --- a/view/sharedcache/core/DSCView.cpp +++ b/view/sharedcache/core/DSCView.cpp @@ -28,8 +28,6 @@ DSCView::DSCView(const std::string& typeName, BinaryView* data, bool parseOnly) DSCView::~DSCView() { - if (!m_parseOnly) - MMappedFileAccessor::CloseAll(GetFile()->GetSessionId()); } enum DSCPlatform { diff --git a/view/sharedcache/core/SharedCache.cpp b/view/sharedcache/core/SharedCache.cpp index bfc06db9..224097f5 100644 --- a/view/sharedcache/core/SharedCache.cpp +++ b/view/sharedcache/core/SharedCache.cpp @@ -215,7 +215,7 @@ uint64_t SharedCache::FastGetBackingCacheCount(BinaryNinja::Ref baseFile; try { - baseFile = MMappedFileAccessor::Open(dscView, dscView->GetFile()->GetSessionId(), dscView->GetFile()->GetOriginalFilename())->lock(); + baseFile = MapFileWithoutApplyingSlide(dscView->GetFile()->GetOriginalFilename()); } catch (...){ LogError("Shared Cache preload: Failed to open file %s", dscView->GetFile()->GetOriginalFilename().c_str()); @@ -278,7 +278,7 @@ void SharedCache::PerformInitialLoad() { m_logger->LogInfo("Performing initial load of Shared Cache"); auto path = m_dscView->GetFile()->GetOriginalFilename(); - auto baseFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), path)->lock(); + auto baseFile = MapFileWithoutApplyingSlide(path); m_viewSpecificState->progress = LoadProgressLoadingCaches; @@ -363,7 +363,7 @@ void SharedCache::PerformInitialLoad() for (auto address : addresses) { i++; - auto vm = GetVMMap(true); + auto vm = GetVMMap(); auto machoHeader = SharedCache::LoadHeaderForAddress(vm, address, "dyld_shared_cache_branch_islands_" + std::to_string(i)); if (machoHeader) { @@ -451,7 +451,7 @@ void SharedCache::PerformInitialLoad() subCachePath = path + "." + entry.fileExtension; subCacheFilename = mainFileName + "." + entry.fileExtension; } - auto subCacheFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), subCachePath)->lock(); + auto subCacheFile = MapFileWithoutApplyingSlide(subCachePath); dyld_cache_header subCacheHeader {}; uint64_t headerSize = subCacheFile->ReadUInt32(16); @@ -537,7 +537,7 @@ void SharedCache::PerformInitialLoad() { auto subCachePath = path + "." + std::to_string(i); auto subCacheFilename = mainFileName + "." + std::to_string(i); - auto subCacheFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), subCachePath)->lock(); + auto subCacheFile = MapFileWithoutApplyingSlide(subCachePath); dyld_cache_header subCacheHeader {}; uint64_t headerSize = subCacheFile->ReadUInt32(16); @@ -582,7 +582,7 @@ void SharedCache::PerformInitialLoad() // Load .symbols subcache try { auto subCachePath = path + ".symbols"; - auto subCacheFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), subCachePath)->lock(); + auto subCacheFile = MapFileWithoutApplyingSlide(subCachePath); dyld_cache_header subCacheHeader {}; uint64_t headerSize = subCacheFile->ReadUInt32(16); @@ -679,7 +679,7 @@ void SharedCache::PerformInitialLoad() subCacheFilename = mainFileName + "." + entry.fileExtension; } - auto subCacheFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), subCachePath)->lock(); + auto subCacheFile = MapFileWithoutApplyingSlide(subCachePath); dyld_cache_header subCacheHeader {}; uint64_t headerSize = subCacheFile->ReadUInt32(16); @@ -738,7 +738,7 @@ void SharedCache::PerformInitialLoad() try { auto subCachePath = path + ".symbols"; - auto subCacheFile = MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), subCachePath)->lock(); + auto subCacheFile = MapFileWithoutApplyingSlide(subCachePath); dyld_cache_header subCacheHeader {}; uint64_t headerSize = subCacheFile->ReadUInt32(16); if (subCacheFile->ReadUInt32(16) > sizeof(dyld_cache_header)) @@ -774,7 +774,7 @@ void SharedCache::PerformInitialLoad() // We have set up enough metadata to map VM now. - auto vm = GetVMMap(true); + auto vm = GetVMMap(); if (!vm) { m_logger->LogError("Failed to map VM pages for Shared Cache on initial load, this is fatal."); @@ -973,21 +973,16 @@ void SharedCache::PerformInitialLoad() m_viewSpecificState->progress = LoadProgressFinished; } -std::shared_ptr SharedCache::GetVMMap(bool mapPages) +std::shared_ptr SharedCache::GetVMMap() { std::shared_ptr vm = std::make_shared(0x1000); - if (mapPages) - { - for (const auto& cache : State().backingCaches) - { - for (const auto& mapping : cache.mappings) - { - vm->MapPages(m_dscView, m_dscView->GetFile()->GetSessionId(), mapping.address, mapping.fileOffset, mapping.size, cache.path, - [this, vm=vm](std::shared_ptr mmap){ - ParseAndApplySlideInfoForFile(mmap); - }); - } + for (const auto& cache : State().backingCaches) { + for (const auto& mapping : cache.mappings) { + vm->MapPages(m_dscView, m_dscView->GetFile()->GetSessionId(), mapping.address, mapping.fileOffset, mapping.size, cache.path, + [this, vm=vm](std::shared_ptr mmap){ + ParseAndApplySlideInfoForFile(mmap); + }); } } @@ -1565,7 +1560,6 @@ bool SharedCache::LoadSectionAtAddress(uint64_t address) } m_logger->LogInfo("Loading stub island %s @ 0x%llx", stubIsland.prettyName.c_str(), stubIsland.start); auto targetFile = vm->MappingAtAddress(stubIsland.start).first.fileAccessor->lock(); - ParseAndApplySlideInfoForFile(targetFile); auto reader = VMReader(vm); auto buff = reader.ReadBuffer(stubIsland.start, stubIsland.size); m_dscView->GetMemoryMap()->AddDataMemoryRegion(stubIsland.prettyName, stubIsland.start, buff, SegmentReadable | SegmentExecutable); @@ -1594,7 +1588,6 @@ bool SharedCache::LoadSectionAtAddress(uint64_t address) } m_logger->LogInfo("Loading dyld data %s", dyldData.prettyName.c_str()); auto targetFile = vm->MappingAtAddress(dyldData.start).first.fileAccessor->lock(); - ParseAndApplySlideInfoForFile(targetFile); auto reader = VMReader(vm); auto buff = reader.ReadBuffer(dyldData.start, dyldData.size); m_dscView->GetMemoryMap()->AddDataMemoryRegion(dyldData.prettyName, dyldData.start, buff, SegmentReadable); @@ -1623,7 +1616,6 @@ bool SharedCache::LoadSectionAtAddress(uint64_t address) } m_logger->LogInfo("Loading non-image region %s", region.prettyName.c_str()); auto targetFile = vm->MappingAtAddress(region.start).first.fileAccessor->lock(); - ParseAndApplySlideInfoForFile(targetFile); auto reader = VMReader(vm); auto buff = reader.ReadBuffer(region.start, region.size); m_dscView->GetMemoryMap()->AddDataMemoryRegion(region.prettyName, region.start, buff, region.flags); @@ -1652,7 +1644,6 @@ bool SharedCache::LoadSectionAtAddress(uint64_t address) m_logger->LogDebug("Partial loading image %s", targetHeader.installName.c_str()); auto targetFile = vm->MappingAtAddress(targetSegment->start).first.fileAccessor->lock(); - ParseAndApplySlideInfoForFile(targetFile); auto buff = reader.ReadBuffer(targetSegment->start, targetSegment->size); m_dscView->GetMemoryMap()->AddDataMemoryRegion(targetSegment->prettyName, targetSegment->start, buff, targetSegment->flags); @@ -1803,8 +1794,6 @@ bool SharedCache::LoadImageWithInstallName(std::string installName, bool skipObj } auto targetFile = vm->MappingAtAddress(region.start).first.fileAccessor->lock(); - ParseAndApplySlideInfoForFile(targetFile); - auto buff = reader.ReadBuffer(region.start, region.size); region.loaded = true; @@ -2847,12 +2836,7 @@ std::vector>> SharedCache::LoadAllSymbolsAndW auto header = HeaderForAddress(img.headerLocation); auto exportList = GetExportListForHeader(*header, [&]() { try { - auto mapping = MMappedFileAccessor::Open( - m_dscView, - m_dscView->GetFile()->GetSessionId(), - header->exportTriePath - )->lock(); - return mapping; + return MapFile(header->exportTriePath); } catch (...) { @@ -2970,7 +2954,7 @@ void SharedCache::FindSymbolAtAddrAndApplyToAddr( auto exportList = GetExportListForHeader(*header, [&]() { try { - return MMappedFileAccessor::Open(m_dscView, m_dscView->GetFile()->GetSessionId(), header->exportTriePath)->lock(); + return MapFile(header->exportTriePath); } catch (...) { @@ -3307,7 +3291,7 @@ extern "C" if (cache->object) { try { - auto vm = cache->object->GetVMMap(true); + auto vm = cache->object->GetVMMap(); auto viewImageHeaders = cache->object->AllImageHeaders(); *count = viewImageHeaders.size(); BNDSCImage* images = new BNDSCImage[viewImageHeaders.size()]; @@ -3386,7 +3370,7 @@ extern "C" BNDSCMemoryUsageInfo BNDSCViewGetMemoryUsageInfo() { BNDSCMemoryUsageInfo info; - info.mmapRefs = mmapCount.load(); + info.mmapRefs = MMapCount(); info.sharedCacheRefs = sharedCacheReferences.load(); return info; } @@ -3695,4 +3679,16 @@ size_t SharedCache::GetObjCRelativeMethodBaseAddress(const VMReader& reader) con return 0; } +std::shared_ptr SharedCache::MapFile(const std::string& path) +{ + return MMappedFileAccessor:: + Open(m_dscView, m_dscView->GetFile()->GetSessionId(), path, [this](std::shared_ptr mmap) { + ParseAndApplySlideInfoForFile(mmap); + })->lock(); +} + +std::shared_ptr SharedCache::MapFileWithoutApplyingSlide(const std::string& path) { + return std::make_shared(path); +} + } // namespace SharedCacheCore diff --git a/view/sharedcache/core/SharedCache.h b/view/sharedcache/core/SharedCache.h index 1b3f8261..1ecfd885 100644 --- a/view/sharedcache/core/SharedCache.h +++ b/view/sharedcache/core/SharedCache.h @@ -588,13 +588,14 @@ namespace SharedCacheCore { void DeserializeFromRawView(); public: - std::shared_ptr GetVMMap(bool mapPages = true); + std::shared_ptr GetVMMap(); static SharedCache* GetFromDSCView(BinaryNinja::Ref dscView); static uint64_t FastGetBackingCacheCount(BinaryNinja::Ref dscView); bool SaveToDSCView(); void ParseAndApplySlideInfoForFile(std::shared_ptr file); + std::optional GetImageStart(std::string installName); std::optional HeaderForAddress(uint64_t); bool LoadImageWithInstallName(std::string installName, bool skipObjC); @@ -655,6 +656,9 @@ private: // Must be called before first access to `MutableState()` after the state // is loaded from the cache. Can safely be called multiple times. void WillMutateState(); + + std::shared_ptr MapFile(const std::string& path); + static std::shared_ptr MapFileWithoutApplyingSlide(const std::string& path); }; } diff --git a/view/sharedcache/core/VM.cpp b/view/sharedcache/core/VM.cpp index 4a3acf66..deab7b0f 100644 --- a/view/sharedcache/core/VM.cpp +++ b/view/sharedcache/core/VM.cpp @@ -50,17 +50,15 @@ #include #endif -void VMShutdown() -{ - std::unique_lock lock2(fileAccessorsMutex); - std::unique_lock lock(fileAccessorDequeMutex); +static std::atomic mmapCount = 0; - // This will trigger the deallocation logic for these. - // It is background threaded to avoid a deadlock on exit. - fileAccessorReferenceHolder.clear(); - fileAccessors.clear(); +uint64_t MMapCount() +{ + return mmapCount; } +void VMShutdown() {} + std::string ResolveFilePath(BinaryNinja::Ref dscView, const std::string& path) { @@ -203,92 +201,121 @@ void MMAP::Unmap() #endif } +FileAccessorCache& FileAccessorCache::Shared() +{ + static FileAccessorCache& shared = *new FileAccessorCache; + return shared; +}; + +FileAccessorCache::FileAccessorCache() = default; -std::shared_ptr MMappedFileAccessor::Open(BinaryNinja::Ref dscView, const uint64_t sessionID, const std::string &path, std::function)> postAllocationRoutine) +void FileAccessorCache::SetCacheSize(uint64_t cacheSize) { - std::scoped_lock lock(fileAccessorsMutex); - if (auto it = fileAccessors.find(path); it != fileAccessors.end()) { - return it->second; + m_cacheSize = cacheSize; + EvictFromCacheIfNeeded(); +} + +void FileAccessorCache::RecordAccess(const std::string& path) +{ + auto it = m_accessors.find(path); + if (it == m_accessors.end()) { + return; + } + + // Move the entry for `path` to the end of `m_leastRecentlyOpened`. + m_leastRecentlyOpened.splice(m_leastRecentlyOpened.end(), m_leastRecentlyOpened, it->second.second); +} + +void FileAccessorCache::EvictFromCacheIfNeeded() +{ + std::lock_guard lock(m_mutex); + while (m_accessors.size() > m_cacheSize) { + m_accessors.erase(m_leastRecentlyOpened.front()); + m_leastRecentlyOpened.pop_front(); } +} - auto fileAcccessor = std::make_shared( - path, - // Allocator logic for the SelfAllocatingWeakPtr - [path=path, sessionID=sessionID, dscView](){ - std::unique_lock _lock(fileAccessorDequeMutex); +std::shared_ptr FileAccessorCache::OpenLazily(BinaryNinja::Ref dscView, + const uint64_t sessionID, const std::string& path, + std::function)> postAllocationRoutine) +{ + std::lock_guard lock(m_mutex); + if (auto it = m_lazyAccessors.find(path); it != m_lazyAccessors.end()) { + RecordAccess(path); + return it->second; + } - // Iterate through held references and start removing them until we can get a file pointer - // FIXME: This could clear all currently used file pointers and still not get one. FIX! - // We should probably use a condition variable here to wait for a file pointer to be released!!! - for (auto& [_, fileAccessorDeque] : fileAccessorReferenceHolder) - { - if (fileAccessorSemaphore.try_acquire()) - break; - fileAccessorDeque.pop_front(); + auto accessor = std::make_shared(path, + [=, dscView = std::move(dscView), postAllocationRoutine = std::move(postAllocationRoutine)]( + const std::string& path) { + auto accessor = Open(dscView, sessionID, path); + if (postAllocationRoutine) { + postAllocationRoutine(accessor); } - - mmapCount++; - _lock.unlock(); - auto accessor = std::shared_ptr(new MMappedFileAccessor(ResolveFilePath(dscView, path)), [](MMappedFileAccessor* accessor){ - // worker thread or we can deadlock on exit here. - BinaryNinja::WorkerEnqueue([accessor](){ - fileAccessorSemaphore.release(); - mmapCount--; - if (fileAccessors.count(accessor->m_path)) - { - std::scoped_lock lock(fileAccessorsMutex); - fileAccessors.erase(accessor->m_path); - } - delete accessor; - }, "MMappedFileAccessor Destructor"); - }); - _lock.lock(); - // If some background thread has managed to try and open a file when the BV was already closed, - // we can still give them the file they want so they dont crash, but as soon as they let go it's gone. - if (!blockedSessionIDs.count(sessionID)) - fileAccessorReferenceHolder[sessionID].push_back(accessor); return accessor; - }, - [postAllocationRoutine=postAllocationRoutine](std::shared_ptr accessor){ - if (postAllocationRoutine) - postAllocationRoutine(std::move(accessor)); }); - fileAccessors.insert_or_assign(path, fileAcccessor); - return fileAcccessor; + + m_lazyAccessors.insert_or_assign(path, accessor); + return accessor; } +std::shared_ptr FileAccessorCache::Open( + BinaryNinja::Ref dscView, const uint64_t sessionID, const std::string& path) +{ + EvictFromCacheIfNeeded(); + + mmapCount++; + auto accessor = std::shared_ptr(new MMappedFileAccessor(ResolveFilePath(dscView, path)), + [this](MMappedFileAccessor* accessor) { Close(accessor); }); + + std::lock_guard lock(m_mutex); + auto [it, inserted] = m_accessors.insert({path, {accessor, m_leastRecentlyOpened.end()}}); + if (inserted) { + m_leastRecentlyOpened.push_back(path); + it->second.second = std::prev(it->second.second); + } else { + RecordAccess(path); + } -void MMappedFileAccessor::CloseAll(const uint64_t sessionID) + return accessor; +} + +void FileAccessorCache::Close(MMappedFileAccessor* accessor) { - blockedSessionIDs.insert(sessionID); - if (fileAccessorReferenceHolder.count(sessionID) == 0) - return; - fileAccessorReferenceHolder.erase(sessionID); + mmapCount--; + delete accessor; } +std::shared_ptr MMappedFileAccessor::Open(BinaryNinja::Ref dscView, + const uint64_t sessionID, const std::string& path, + std::function)> postAllocationRoutine) +{ + return FileAccessorCache::Shared().OpenLazily(dscView, sessionID, path, std::move(postAllocationRoutine)); +} void MMappedFileAccessor::InitialVMSetup() { - // check for BN_SHAREDCACHE_FP_MAX - // if it exists, set maxFPLimit to that value - maxFPLimit = 0; - if (auto env = getenv("BN_SHAREDCACHE_FP_MAX"); env) - { - // FIXME behav on 0 here is unintuitive, '0123' will interpret as octal and be 83 according to manpage. meh. - maxFPLimit = strtoull(env, nullptr, 0); - if (maxFPLimit < 10) + static std::once_flag once; + std::call_once(once, []{ + // check for BN_SHAREDCACHE_FP_MAX + // if it exists, set maxFPLimit to that value + unsigned long long maxFPLimit = 0; + if (auto env = getenv("BN_SHAREDCACHE_FP_MAX"); env) { - BinaryNinja::LogWarn("BN_SHAREDCACHE_FP_MAX set to below 10. A value of at least 10 is recommended for performant analysis on SharedCache Binaries."); + // FIXME behav on 0 here is unintuitive, '0123' will interpret as octal and be 83 according to manpage. meh. + maxFPLimit = strtoull(env, nullptr, 0); + if (maxFPLimit < 10) + { + BinaryNinja::LogWarn("BN_SHAREDCACHE_FP_MAX set to below 10. A value of at least 10 is recommended for performant analysis on SharedCache Binaries."); + } + if (maxFPLimit == 0) + { + BinaryNinja::LogError("BN_SHAREDCACHE_FP_MAX set to 0. Adjusting to 1"); + maxFPLimit = 1; + } } - if (maxFPLimit == 0) + else { - BinaryNinja::LogError("BN_SHAREDCACHE_FP_MAX set to 0. Adjusting to 1"); - maxFPLimit = 1; - } - } - else - { - if (maxFPLimit < 10) { #ifdef _MSC_VER // It is not _super_ clear what the max file pointer limit is on windows, // but to my understanding, we are using the windows API to map files, @@ -297,15 +324,30 @@ void MMappedFileAccessor::InitialVMSetup() // parallelize sharedcache processing on in terms of FP usage concerns maxFPLimit = 0x1000000; #else - // unix in comparison will likely have a very small limit, especially mac, necessitating all of this consideration + // The soft file descriptor limit on Linux and Mac is a lot lower than + // on Windows (1024 for Linux, 256 for Mac). Recent iOS shared caches + // have 60+ files which may not leave much headroom if a user opens + // more than one at a time. Attempt to increase the file descriptor + // limit to 1024, and limit ourselves to caching half of them as a + // memory vs performance trade-off (closing and re-opening a file + // requires parsing and applying the slide information again). + constexpr rlim_t TargetFileDescriptorLimit = 1024; struct rlimit rlim; getrlimit(RLIMIT_NOFILE, &rlim); + unsigned long long previousLimit = rlim.rlim_cur; + if (rlim.rlim_cur < TargetFileDescriptorLimit) { + rlim.rlim_cur = std::min(TargetFileDescriptorLimit, rlim.rlim_max); + if (setrlimit(RLIMIT_NOFILE, &rlim) < 0) { + perror("setrlimit(RLIMIT_NOFILE)"); + rlim.rlim_cur = previousLimit; + } + } maxFPLimit = rlim.rlim_cur / 2; #endif } - } - BinaryNinja::LogInfo("Shared Cache processing initialized with a max file pointer limit of 0x%llx", maxFPLimit); - fileAccessorSemaphore.set_count(maxFPLimit); + BinaryNinja::LogInfo("Shared Cache processing initialized with a max file descriptor limit of %lld", maxFPLimit); + FileAccessorCache::Shared().SetCacheSize(maxFPLimit); + }); } @@ -486,7 +528,8 @@ void VM::MapPages(BinaryNinja::Ref dscView, uint64_t se throw MappingPageAlignmentException(); } - auto accessor = MMappedFileAccessor::Open(std::move(dscView), sessionID, filePath, postAllocationRoutine); + auto accessor = + MMappedFileAccessor::Open(std::move(dscView), sessionID, filePath, std::move(postAllocationRoutine)); auto [it, inserted] = m_map.insert_or_assign({vm_address, vm_address + size}, PageMapping(std::move(accessor), fileoff)); if (m_safe && !inserted) { diff --git a/view/sharedcache/core/VM.h b/view/sharedcache/core/VM.h index dd4f3842..402df196 100644 --- a/view/sharedcache/core/VM.h +++ b/view/sharedcache/core/VM.h @@ -4,62 +4,26 @@ #ifndef SHAREDCACHE_VM_H #define SHAREDCACHE_VM_H + #include -#include + +#include +#include +#include void VMShutdown(); std::string ResolveFilePath(BinaryNinja::Ref dscView, const std::string& path); -class counting_semaphore { -public: - explicit counting_semaphore(int count = 0) : count_(count) {} - - void release(int update = 1) { - std::unique_lock lock(mutex_); - count_ += update; - cv_.notify_all(); - } - - void acquire() { - std::unique_lock lock(mutex_); - cv_.wait(lock, [this]() { return count_ > 0; }); - --count_; - } - - bool try_acquire() { - std::unique_lock lock(mutex_); - if (count_ > 0) { - --count_; - return true; - } - return false; - } - - void set_count(int new_count) { - std::unique_lock lock(mutex_); - count_ = new_count; - cv_.notify_all(); - } - -private: - std::mutex mutex_; - std::condition_variable cv_; - int count_; -}; - - template class SelfAllocatingWeakPtr { public: - SelfAllocatingWeakPtr(std::function()> allocator, std::function)> postAlloc) - : allocator(allocator), postAlloc(postAlloc) {} + SelfAllocatingWeakPtr(std::function()> allocator) : allocator(allocator) {} std::shared_ptr lock() { std::shared_ptr sharedPtr = weakPtr.lock(); if (!sharedPtr) { sharedPtr = allocator(); - postAlloc(sharedPtr); weakPtr = sharedPtr; } return sharedPtr; @@ -71,8 +35,7 @@ public: private: std::weak_ptr weakPtr; // Weak reference to the object - std::function()> allocator; // Function to recreate the object - std::function)> postAlloc; // Function to call after the object is allocated + std::function()> allocator; // Function to recreate the object }; @@ -106,26 +69,20 @@ class MMAP { class LazyMappedFileAccessor : public SelfAllocatingWeakPtr { public: - LazyMappedFileAccessor(std::string filePath, std::function()> allocator, - std::function)> postAlloc) - : SelfAllocatingWeakPtr(std::move(allocator), std::move(postAlloc)), m_filePath(std::move(filePath)) { - } + LazyMappedFileAccessor( + std::string filePath, std::function(const std::string&)> allocator) : + SelfAllocatingWeakPtr([this, allocator = std::move(allocator)] { return allocator(m_filePath); }), + m_filePath(std::move(filePath)) + {} + ~LazyMappedFileAccessor() = default; - std::string_view filePath() const { return m_filePath; } + std::string_view filePath() const { return m_filePath; } private: std::string m_filePath; }; -static uint64_t maxFPLimit; -static std::mutex fileAccessorDequeMutex; -static std::unordered_map>> fileAccessorReferenceHolder; -static std::set blockedSessionIDs; -static std::mutex fileAccessorsMutex; -static std::unordered_map> fileAccessors; -static counting_semaphore fileAccessorSemaphore(0); - -static std::atomic mmapCount = 0; +uint64_t MMapCount(); class MMappedFileAccessor { std::string m_path; @@ -142,9 +99,9 @@ public: static void InitialVMSetup(); - std::string Path() const { return m_path; }; + const std::string& Path() const { return m_path; }; - size_t Length() const { return m_mmap.len; }; + size_t Length() const { return m_mmap.len; }; void *Data() const { return m_mmap._mmap; }; @@ -201,6 +158,39 @@ public: T Read(size_t address); }; +class FileAccessorCache +{ +public: + static FileAccessorCache& Shared(); + + std::shared_ptr OpenLazily(BinaryNinja::Ref dscView, + const uint64_t sessionID, const std::string& path, + std::function)> postAllocationRoutine); + + void SetCacheSize(uint64_t size); + +private: + FileAccessorCache(); + + std::shared_ptr Open( + BinaryNinja::Ref dscView, const uint64_t sessionID, const std::string& path); + + void Close(MMappedFileAccessor* accessor); + + void RecordAccess(const std::string& path); + void EvictFromCacheIfNeeded(); + + std::mutex m_mutex; + std::unordered_map> m_lazyAccessors; + + // Ordered from least recently opened (front) to most recently opened (end). + std::list m_leastRecentlyOpened; + std::unordered_map, std::list::iterator>> + m_accessors; + + uint64_t m_cacheSize = 8; +}; + struct PageMapping { std::shared_ptr fileAccessor; size_t fileOffset; -- cgit v1.3.1