diff options
| author | Peter LaFosse <peter@vector35.com> | 2021-02-09 20:39:28 -0500 |
|---|---|---|
| committer | Peter LaFosse <peter@vector35.com> | 2021-03-02 07:50:56 -0500 |
| commit | 1eafe9eeb83962071ecffaebb0bc98dc79f2f5c6 (patch) | |
| tree | f242857ceffdf76576bb87745cf096ffefc71b93 | |
| parent | c07fe0fea819f681621ff60e336f079494f6bfef (diff) | |
Add ability to install python dependencies
Refactor plugin repo loading into scripting provider.
Find python binary for given python library
| -rw-r--r-- | binaryninjaapi.cpp | 15 | ||||
| -rw-r--r-- | binaryninjaapi.h | 14 | ||||
| -rw-r--r-- | binaryninjacore.h | 19 | ||||
| -rw-r--r-- | pluginmanager.cpp | 11 | ||||
| -rw-r--r-- | python/__init__.py | 49 | ||||
| -rw-r--r-- | python/pluginmanager.py | 8 | ||||
| -rw-r--r-- | python/scriptingprovider.py | 163 | ||||
| -rw-r--r-- | scriptingprovider.cpp | 50 |
8 files changed, 247 insertions, 82 deletions
diff --git a/binaryninjaapi.cpp b/binaryninjaapi.cpp index 3b4974fc..6b2497a9 100644 --- a/binaryninjaapi.cpp +++ b/binaryninjaapi.cpp @@ -385,17 +385,4 @@ map<string, uint64_t> BinaryNinja::GetMemoryUsageInfo() result[info[i].name] = info[i].value; BNFreeMemoryUsageInfo(info, count); return result; -} - - -vector<string> BinaryNinja::GetRegisteredPluginLoaders() -{ - size_t count = 0; - char** loaders = BNGetRegisteredPluginLoaders(&count); - vector<string> result; - result.reserve(count); - for (size_t i = 0; i < count; i++) - result.push_back(loaders[i]); - BNFreeRegisteredPluginLoadersList(loaders, count); - return result; -} +}
\ No newline at end of file diff --git a/binaryninjaapi.h b/binaryninjaapi.h index cd52dc44..bfc441d6 100644 --- a/binaryninjaapi.h +++ b/binaryninjaapi.h @@ -756,7 +756,6 @@ __attribute__ ((format (printf, 1, 2))) std::string GetUniqueIdentifierString(); std::map<std::string, uint64_t> GetMemoryUsageInfo(); - std::vector<std::string> GetRegisteredPluginLoaders(); class DataBuffer { @@ -5010,20 +5009,27 @@ __attribute__ ((format (printf, 1, 2))) class ScriptingProvider: public StaticCoreRefCountObject<BNScriptingProvider> { std::string m_nameForRegister; + std::string m_apiNameForRegister; protected: - ScriptingProvider(const std::string& name); + ScriptingProvider(const std::string& name, const std::string& apiName); ScriptingProvider(BNScriptingProvider* provider); static BNScriptingInstance* CreateInstanceCallback(void* ctxt); + static bool LoadModuleCallback(void* ctxt, const char* repository, const char* module, bool force); + static bool InstallModulesCallback(void* ctxt, const char* modules); public: virtual Ref<ScriptingInstance> CreateNewInstance() = 0; + virtual bool LoadModule(const std::string& repository, const std::string& module, bool force) = 0; + virtual bool InstallModules(const std::string& modules) = 0; std::string GetName(); + std::string GetAPIName(); static std::vector<Ref<ScriptingProvider>> GetList(); static Ref<ScriptingProvider> GetByName(const std::string& name); + static Ref<ScriptingProvider> GetByAPIName(const std::string& apiName); static void Register(ScriptingProvider* provider); }; @@ -5032,6 +5038,8 @@ __attribute__ ((format (printf, 1, 2))) public: CoreScriptingProvider(BNScriptingProvider* provider); virtual Ref<ScriptingInstance> CreateNewInstance() override; + virtual bool LoadModule(const std::string& repository, const std::string& module, bool force) override; + virtual bool InstallModules(const std::string& modules) override; }; class MainThreadAction: public CoreRefCountObject<BNMainThreadAction, @@ -5159,6 +5167,7 @@ __attribute__ ((format (printf, 1, 2))) std::vector<std::string> GetApis() const; std::vector<std::string> GetInstallPlatforms() const; std::string GetPath() const; + std::string GetDependencies() const; std::string GetPluginDirectory() const; std::string GetAuthor() const; std::string GetDescription() const; @@ -5189,6 +5198,7 @@ __attribute__ ((format (printf, 1, 2))) bool Uninstall(); bool Install(); + bool InstallDependencies(); // `force` ignores optional checks for platform/api compliance bool Enable(bool force); bool Disable(); diff --git a/binaryninjacore.h b/binaryninjacore.h index c46f98c1..0fdb2121 100644 --- a/binaryninjacore.h +++ b/binaryninjacore.h @@ -210,7 +210,6 @@ extern "C" struct BNLinearViewObject; struct BNLinearViewCursor; - typedef bool (*BNLoadPluginCallback)(const char* repoPath, const char* pluginPath, bool force, void* ctx); //! Console log levels enum BNLogLevel @@ -2208,7 +2207,8 @@ extern "C" { void* context; BNScriptingInstance* (*createInstance)(void* ctxt); - + bool (*loadModule)(void* ctxt, const char* repoPath, const char* pluginPath, bool force); + bool (*installModules)(void* ctxt, const char* modules); }; struct BNScriptingOutputListener @@ -4663,14 +4663,18 @@ __attribute__ ((format (printf, 1, 2))) BINARYNINJACOREAPI void BNSetErrorForDownloadInstance(BNDownloadInstance* instance, const char* error); // Scripting providers - BINARYNINJACOREAPI BNScriptingProvider* BNRegisterScriptingProvider(const char* name, + BINARYNINJACOREAPI BNScriptingProvider* BNRegisterScriptingProvider(const char* name, const char* apiName, BNScriptingProviderCallbacks* callbacks); BINARYNINJACOREAPI BNScriptingProvider** BNGetScriptingProviderList(size_t* count); BINARYNINJACOREAPI void BNFreeScriptingProviderList(BNScriptingProvider** providers); BINARYNINJACOREAPI BNScriptingProvider* BNGetScriptingProviderByName(const char* name); + BINARYNINJACOREAPI BNScriptingProvider* BNGetScriptingProviderByAPIName(const char* name); BINARYNINJACOREAPI char* BNGetScriptingProviderName(BNScriptingProvider* provider); + BINARYNINJACOREAPI char* BNGetScriptingProviderAPIName(BNScriptingProvider* provider); BINARYNINJACOREAPI BNScriptingInstance* BNCreateScriptingProviderInstance(BNScriptingProvider* provider); + BINARYNINJACOREAPI bool BNLoadScriptingProviderModule(BNScriptingProvider* provider, const char* repository, const char* module, bool force); + BINARYNINJACOREAPI bool BNInstallScriptingProviderModules(BNScriptingProvider* provider, const char* modules); BINARYNINJACOREAPI BNScriptingInstance* BNInitScriptingInstance(BNScriptingProvider* provider, BNScriptingInstanceCallbacks* callbacks); @@ -4805,6 +4809,7 @@ __attribute__ ((format (printf, 1, 2))) BINARYNINJACOREAPI BNRepoPlugin* BNNewPluginReference(BNRepoPlugin* r); BINARYNINJACOREAPI void BNFreePlugin(BNRepoPlugin* plugin); BINARYNINJACOREAPI const char* BNPluginGetPath(BNRepoPlugin* p); + BINARYNINJACOREAPI const char* BNPluginGetDependencies(BNRepoPlugin* p); BINARYNINJACOREAPI bool BNPluginIsInstalled(BNRepoPlugin* p); BINARYNINJACOREAPI bool BNPluginIsEnabled(BNRepoPlugin* p); BINARYNINJACOREAPI BNPluginStatus BNPluginGetPluginStatus(BNRepoPlugin* p); @@ -4812,6 +4817,7 @@ __attribute__ ((format (printf, 1, 2))) BINARYNINJACOREAPI bool BNPluginEnable(BNRepoPlugin* p, bool force); BINARYNINJACOREAPI bool BNPluginDisable(BNRepoPlugin* p); BINARYNINJACOREAPI bool BNPluginInstall(BNRepoPlugin* p); + BINARYNINJACOREAPI bool BNPluginInstallDependencies(BNRepoPlugin* p); BINARYNINJACOREAPI bool BNPluginUninstall(BNRepoPlugin* p); BINARYNINJACOREAPI bool BNPluginUpdate(BNRepoPlugin* p); BINARYNINJACOREAPI char* BNPluginGetInstallInstructions(BNRepoPlugin* p, const char* platform); @@ -4852,13 +4858,6 @@ __attribute__ ((format (printf, 1, 2))) BINARYNINJACOREAPI BNRepositoryManager* BNGetRepositoryManager(); BINARYNINJACOREAPI BNRepository* BNRepositoryManagerGetDefaultRepository(BNRepositoryManager* r); - BINARYNINJACOREAPI void BNRegisterForPluginLoading( - const char* pluginApiName, - bool (*cb)(const char* repoPath, const char* pluginPath, bool force, void* ctx), // BNLoadPluginCallback - void* ctx); - BINARYNINJACOREAPI bool BNLoadPluginForApi(const char* pluginApiName, const char* repoPath, const char* pluginPath, bool force); - BINARYNINJACOREAPI char** BNGetRegisteredPluginLoaders(size_t* count); - BINARYNINJACOREAPI void BNFreeRegisteredPluginLoadersList(char** pluginLoaders, size_t count); // LLVM Services APIs BINARYNINJACOREAPI void BNLlvmServicesInit(void); diff --git a/pluginmanager.cpp b/pluginmanager.cpp index 8f57f2f8..049555be 100644 --- a/pluginmanager.cpp +++ b/pluginmanager.cpp @@ -20,6 +20,11 @@ string RepoPlugin::GetPath() const RETURN_STRING(BNPluginGetPath(m_object)); } +string RepoPlugin::GetDependencies() const +{ + RETURN_STRING(BNPluginGetDependencies(m_object)); +} + bool RepoPlugin::IsInstalled() const { return BNPluginIsInstalled(m_object); @@ -214,6 +219,12 @@ bool RepoPlugin::Install() } +bool RepoPlugin::InstallDependencies() +{ + return BNPluginInstallDependencies(m_object); +} + + bool RepoPlugin::Enable(bool force) { return BNPluginEnable(m_object, force); diff --git a/python/__init__.py b/python/__init__.py index 8855a395..4c4c412c 100644 --- a/python/__init__.py +++ b/python/__init__.py @@ -26,7 +26,6 @@ import ctypes from time import gmtime import os - from binaryninja.compatibility import * @@ -90,54 +89,6 @@ def get_install_directory(): return core.BNGetInstallDirectory() -_plugin_api_name = "python{}".format(sys.version_info.major) - - -class PluginManagerLoadPluginCallback(object): - """Callback for BNLoadPluginForApi("python{version}", ...), dynamically loads python plugins.""" - def __init__(self): - self.cb = ctypes.CFUNCTYPE( - ctypes.c_bool, - ctypes.c_char_p, - ctypes.c_char_p, - ctypes.c_bool, - ctypes.c_void_p)(self._load_plugin) - - def _load_plugin(self, repo_path, plugin_path, force, ctx): - try: - repo_path = repo_path.decode("utf-8") - plugin_path = plugin_path.decode("utf-8") - repo = RepositoryManager()[repo_path] - plugin = repo[plugin_path] - - if not force and _plugin_api_name not in plugin.api: - raise ValueError("Plugin API name is not " + _plugin_api_name) - - if not force and core.core_platform not in plugin.install_platforms: - raise ValueError("Current platform {} isn't in list of valid platforms for this plugin {}".format( - core.core_platform, plugin.install_platforms)) - if not plugin.installed: - plugin.installed = True - - plugin_full_path = os.path.join(repo.full_path, plugin.path) - if repo.full_path not in sys.path: - sys.path.append(repo.full_path) - if plugin_full_path not in sys.path: - sys.path.append(plugin_full_path) - - __import__(plugin_path) - return True - except KeyError: - log_error("Failed to find python plugin: {}/{}".format(repo_path, plugin_path)) - except ImportError as ie: - log_error("Failed to import python plugin: {}/{}: {}".format(repo_path, plugin_path, ie)) - return False - - -load_plugin = PluginManagerLoadPluginCallback() -core.BNRegisterForPluginLoading(_plugin_api_name, load_plugin.cb, 0) - - class _DestructionCallbackHandler(object): def __init__(self): self._cb = core.BNObjectDestructionCallbacks() diff --git a/python/pluginmanager.py b/python/pluginmanager.py index ea3440f3..a164c4f6 100644 --- a/python/pluginmanager.py +++ b/python/pluginmanager.py @@ -50,6 +50,11 @@ class RepoPlugin(object): return core.BNPluginGetPath(self.handle) @property + def dependencies(self): + """Dependencies required for installing this plugin""" + return core.BNPluginGetDependencies(self.handle) + + @property def installed(self): """Boolean True if the plugin is installed, False otherwise""" return core.BNPluginIsInstalled(self.handle) @@ -61,6 +66,9 @@ class RepoPlugin(object): else: return core.BNPluginUninstall(self.handle) + def install_dependencies(self): + return core.BNPluginInstallDependencies(self.handle) + @property def enabled(self): """Boolean True if the plugin is currently enabled, False otherwise""" diff --git a/python/scriptingprovider.py b/python/scriptingprovider.py index 7a60e6c0..3205826b 100644 --- a/python/scriptingprovider.py +++ b/python/scriptingprovider.py @@ -25,17 +25,19 @@ import ctypes import threading import abc import sys -from binaryninja import bncompleter +import subprocess +from pathlib import Path, PurePath import re # Binary Ninja components import binaryninja +from binaryninja import bncompleter, log from binaryninja import _binaryninjacore as core +from binaryninja.settings import Settings +from binaryninja.pluginmanager import RepositoryManager from binaryninja.enums import ScriptingProviderExecuteResult, ScriptingProviderInputReadyState -from binaryninja import log # 2-3 compatibility -from binaryninja import range from binaryninja import with_metaclass @@ -364,7 +366,9 @@ class ScriptingProvider(with_metaclass(_ScriptingProviderMetaclass, object)): self._cb = core.BNScriptingProviderCallbacks() self._cb.context = 0 self._cb.createInstance = self._cb.createInstance.__class__(self._create_instance) - self.handle = core.BNRegisterScriptingProvider(self.__class__.name, self._cb) + self._cb.loadModule = self._cb.loadModule.__class__(self._load_module) + self._cb.installModules = self._cb.installModules.__class__(self._install_modules) + self.handle = core.BNRegisterScriptingProvider(self.__class__.name, self.__class__.apiName, self._cb) self.__class__._registered_providers.append(self) def _create_instance(self, ctxt): @@ -383,6 +387,12 @@ class ScriptingProvider(with_metaclass(_ScriptingProviderMetaclass, object)): return None return ScriptingInstance(self, handle = result) + def _load_plugin(self, ctx, repo_path, plugin_path, force): + return False + + def _install_modules(self, ctx, modules): + return False + class _PythonScriptingInstanceOutput(object): def __init__(self, orig, is_error): @@ -422,7 +432,7 @@ class _PythonScriptingInstanceOutput(object): def seek(self): pass - def sofspace(self): + def softspace(self): return 0 def truncate(self): @@ -789,10 +799,153 @@ class PythonScriptingInstance(ScriptingInstance): return "" return result + class PythonScriptingProvider(ScriptingProvider): name = "Python" + apiName = f"python{sys.version_info.major}" # Used for plugin compatibility testing instance_class = PythonScriptingInstance + def _load_module(self, ctx, repo_path, module, force): + try: + repo_path = repo_path.decode("utf-8") + module = module.decode("utf-8") + repo = RepositoryManager()[repo_path] + plugin = repo[module] + + if not force and self.apiName not in plugin.api: + raise ValueError(f"Plugin API name is not {self.name}") + + if not force and core.core_platform not in plugin.install_platforms: + raise ValueError(f"Current platform {core.core_platform} isn't in list of "\ + f"valid platforms for this plugin {plugin.install_platforms}") + if not plugin.installed: + plugin.installed = True + + plugin_full_path = str(Path(repo.full_path) / plugin.path) + if repo.full_path not in sys.path: + sys.path.append(repo.full_path) + if plugin_full_path not in sys.path: + sys.path.append(plugin_full_path) + + __import__(module) + return True + except KeyError: + log.log_error(f"Failed to find python plugin: {repo_path}/{module}") + except ImportError as ie: + log.log_error(f"Failed to import python plugin: {repo_path}/{module}: {ie}") + return False + + def _run_args(self, args): + si = None + if sys.platform == "win32": + si = subprocess.STARTUPINFO() + si.dwFlags |= subprocess.STARTF_USESHOWWINDOW + + try: + return (True, subprocess.check_output(args, startupinfo=si).decode("utf-8")) + except SubprocessError as se: + return (False, str(se)) + + + def _bin_version(self, python_bin: str): + return self._run_args([str(python_bin), "-c", "import sys; sys.stdout.write(f'{sys.version_info.major}.{sys.version_info.minor}')"])[1] + + def _get_executable_for_libpython(self, python_lib: str, python_bin: str): + python_lib_version = f"{sys.version_info.major}.{sys.version_info.minor}" + if python_bin is not None and python_bin != "": + python_bin_version = self._bin_version(python_bin) + if python_lib_version != python_bin_version: + return (None, f"Specified Python Binary Override is the wrong version. Expected: {python_lib_version} got: {python_bin_version}") + return (python_bin, "Success") + + using_bundled_python = python_lib is None + si = None + if sys.platform == "darwin": + if using_bundled_python: + return (None, "Failed: Bundled python doesn't support dependency installation. Specify a full python installation in your 'Python Interpreter' and try again") + + python_bin = Path(python_lib).parent / f"bin/python{python_lib_version}" + elif sys.platform == "linux": + class Dl_info(ctypes.Structure): + _fields_ = [ + ("dli_fname", ctypes.c_char_p), + ("dli_fbase", ctypes.c_void_p), + ("dli_sname", ctypes.c_char_p), + ("dli_saddr", ctypes.c_void_p), + ] + def _linked_libpython(): + libdl = ctypes.CDLL(find_library("dl")) + libdl.dladdr.argtypes = [ctypes.c_void_p, ctypes.POINTER(Dl_info)] + libdl.dladdr.restype = ctypes.c_int + dlinfo = Dl_info() + retcode = libdl.dladdr(ctypes.cast(ctypes.pythonapi.Py_GetVersion, ctypes.c_void_p), ctypes.pointer(dlinfo)) + if retcode == 0: # means error + return None + return os.path.realpath(dlinfo.dli_fname.decode()) + if using_bundled_python: + python_lib = _linked_libpython() + if python_lib is None: + return (None, "Failed: No python specified. Specify a full python installation in your 'Python Interpreter' and try again") + + if python_lib == os.path.realpath(sys.executable): + python_bin = python_lib + else: + python_path = Path(python_lib) + for path in python_path.parents: + if path.name in ["lib", "lib64"]: + break + else: + return (None, f"Failed to find python binary from {python_lib}") + + python_bin = path.parent / f"bin/python{python_lib_version}" + else: + if using_bundled_python: + python_bin = Path(get_install_directory()) / "plugins\\python\\python.exe" + else: + python_bin = Path(python_lib).parent / "python.exe" + python_bin_version = self._bin_version(python_bin) + if python_bin_version != python_lib_version: + return (None, f"Failed: Python version not equal {python_bin_version} and {python_lib_version}") + + return (python_bin, "Success") + + def _install_modules(self, ctx, modules): + # This callback should not be called directly it is indirectly + # executed binary ninja is executed with --pip option + python_lib = Settings().get_string("python.interpreter") + python_bin_override = Settings().get_string("python.binaryOverride") + python_bin, status = self._get_executable_for_libpython(python_lib, python_bin_override) + if sys.platform == "darwin" and not any([python_bin, python_lib, python_bin_override]): + log.log_error(f"Plugin requirement installation unsupported on MacOS with bundled Python: {status} Please specify a path to a python library in the 'Python Interpreter' setting") + return False + elif python_bin is None: + log.log_error(f"Unable to discover python executable required for installing python modules: {status} Please specify a path to a python binary in the 'Python Path Override'") + return False + + python_bin_version = subprocess.check_output([python_bin, "-c", "import sys; sys.stdout.write(f'{sys.version_info.major}.{sys.version_info.minor}')"]).decode("utf-8") + python_lib_version = f"{sys.version_info.major}.{sys.version_info.minor}" + if (python_bin_version != python_lib_version): + log.log_error(f"Python Binary Setting {python_bin_version} incompatible with python library {python_lib_version}") + return False + + args = [str(python_bin), "-m", "pip", "--isolated", "--disable-pip-version-check"] + proxy_settings = Settings().get_string("downloadClient.httpsProxy") + if proxy_settings: + args.extend(["--proxy", proxy_settings]) + + args.append("install") + args.append("--verbose") + venv = Settings().get_string("python.virtualenv") + if venv is not None and venv.endswith("site-packages") and Path(venv).is_dir(): + args.extend(["--target", venv]) + + args.extend(filter(len, modules.decode("utf-8").split("\n"))) + log.log_info(f"Running pip {args}") + status, result = self._run_args(args) + if not status: + log.log_error(f"Error while attempting to install requirements {result}") + return status + PythonScriptingProvider().register() # Wrap stdin/stdout/stderr for Python scripting provider implementation diff --git a/scriptingprovider.cpp b/scriptingprovider.cpp index ed010100..4c11ccc4 100644 --- a/scriptingprovider.cpp +++ b/scriptingprovider.cpp @@ -281,7 +281,7 @@ std::string CoreScriptingInstance::CompleteInput(const std::string& text, uint64 } -ScriptingProvider::ScriptingProvider(const string& name): m_nameForRegister(name) +ScriptingProvider::ScriptingProvider(const string& name, const string& apiName): m_nameForRegister(name), m_apiNameForRegister(apiName) { } @@ -300,6 +300,20 @@ BNScriptingInstance* ScriptingProvider::CreateInstanceCallback(void* ctxt) } +bool ScriptingProvider::LoadModuleCallback(void* ctxt, const char* repository, const char* module, bool force) +{ + ScriptingProvider* provider = (ScriptingProvider*)ctxt; + return BNLoadScriptingProviderModule(provider->GetObject(), repository, module, force); +} + + +bool ScriptingProvider::InstallModulesCallback(void* ctxt, const char* modules) +{ + ScriptingProvider* provider = (ScriptingProvider*)ctxt; + return BNInstallScriptingProviderModules(provider->GetObject(), modules); +} + + string ScriptingProvider::GetName() { char* providerNameRaw = BNGetScriptingProviderName(m_object); @@ -309,6 +323,15 @@ string ScriptingProvider::GetName() } +string ScriptingProvider::GetAPIName() +{ + char* providerNameRaw = BNGetScriptingProviderAPIName(m_object); + string providerName(providerNameRaw); + BNFreeString(providerNameRaw); + return providerName; +} + + vector<Ref<ScriptingProvider>> ScriptingProvider::GetList() { size_t count; @@ -330,12 +353,23 @@ Ref<ScriptingProvider> ScriptingProvider::GetByName(const string& name) } +Ref<ScriptingProvider> ScriptingProvider::GetByAPIName(const string& name) +{ + BNScriptingProvider* result = BNGetScriptingProviderByAPIName(name.c_str()); + if (!result) + return nullptr; + return new CoreScriptingProvider(result); +} + + void ScriptingProvider::Register(ScriptingProvider* provider) { BNScriptingProviderCallbacks cb; cb.context = provider; cb.createInstance = CreateInstanceCallback; - provider->m_object = BNRegisterScriptingProvider(provider->m_nameForRegister.c_str(), &cb); + cb.loadModule = LoadModuleCallback; + cb.installModules = InstallModulesCallback; + provider->m_object = BNRegisterScriptingProvider(provider->m_nameForRegister.c_str(), provider->m_apiNameForRegister.c_str(), &cb); } @@ -351,3 +385,15 @@ Ref<ScriptingInstance> CoreScriptingProvider::CreateNewInstance() return nullptr; return new CoreScriptingInstance(result); } + + +bool CoreScriptingProvider::LoadModule(const std::string& repository, const std::string& module, bool force) +{ + return BNLoadScriptingProviderModule(m_object, repository.c_str(), module.c_str(), force); +} + + +bool CoreScriptingProvider::InstallModules(const std::string& modules) +{ + return BNInstallScriptingProviderModules(m_object, modules.c_str()); +} |
