From 811544620f903b137aeb7660183ca9d6bb0d7590 Mon Sep 17 00:00:00 2001 From: Brian Potchik Date: Wed, 20 Jun 2018 16:03:27 -0400 Subject: Add DownloadProvider Support. --- python/downloadprovider.py | 210 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 python/downloadprovider.py (limited to 'python/downloadprovider.py') diff --git a/python/downloadprovider.py b/python/downloadprovider.py new file mode 100644 index 00000000..4756fd90 --- /dev/null +++ b/python/downloadprovider.py @@ -0,0 +1,210 @@ +# Copyright (c) 2015-2018 Vector 35 LLC +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +import abc +import code +import ctypes +import sys +import traceback + +try: + from urllib.parse import urlparse, urlencode + from urllib.request import urlopen, Request, build_opener, install_opener, ProxyHandler + from urllib.error import HTTPError, URLError +except ImportError: + from urlparse import urlparse + from urllib import urlencode + from urllib2 import urlopen, Request, build_opener, install_opener, ProxyHandler, HTTPError, URLError + +# Binary Ninja Components +import _binaryninjacore as core +from binaryninja.setting import Setting +import startup +import log + + +class DownloadInstance(object): + def __init__(self, provider, handle = None): + if handle is None: + self._cb = core.BNDownloadInstanceCallbacks() + self._cb.context = 0 + self._cb.destroyInstance = self._cb.destroyInstance.__class__(self._destroy_instance) + self._cb.performRequest = self._cb.performRequest.__class__(self._perform_request) + self.handle = core.BNInitDownloadInstance(provider.handle, self._cb) + else: + self.handle = core.handle_of_type(handle, core.BNDownloadInstance) + self._outputCallbacks = None + + def __del__(self): + core.BNFreeDownloadInstance(self.handle) + + def _destroy_instance(self, ctxt): + try: + self.perform_destroy_instance() + except: + log.log_error(traceback.format_exc()) + + def _perform_request(self, ctxt, url): + try: + return self.perform_request(ctxt, url) + except: + log.log_error(traceback.format_exc()) + return -1 + + @abc.abstractmethod + def perform_destroy_instance(self): + raise NotImplementedError + + @abc.abstractmethod + def perform_request(self, ctxt, url): + raise NotImplementedError + + def perform_request(self, url, callbacks): + return core.BNPerformDownloadRequest(self.handle, url, callbacks) + + +class _DownloadProviderMetaclass(type): + @property + def list(self): + """List all DownloadProvider types (read-only)""" + startup._init_plugins() + count = ctypes.c_ulonglong() + types = core.BNGetDownloadProviderList(count) + result = [] + for i in xrange(0, count.value): + result.append(DownloadProvider(types[i])) + core.BNFreeDownloadProviderList(types) + return result + + def __iter__(self): + startup._init_plugins() + count = ctypes.c_ulonglong() + types = core.BNGetDownloadProviderList(count) + try: + for i in xrange(0, count.value): + yield DownloadProvider(types[i]) + finally: + core.BNFreeDownloadProviderList(types) + + def __getitem__(self, value): + startup._init_plugins() + provider = core.BNGetDownloadProviderByName(str(value)) + if provider is None: + raise KeyError("'%s' is not a valid download provider" % str(value)) + return DownloadProvider(provider) + + def __setattr__(self, name, value): + try: + type.__setattr__(self, name, value) + except AttributeError: + raise AttributeError("attribute '%s' is read only" % name) + + +class DownloadProvider(object): + __metaclass__ = _DownloadProviderMetaclass + name = None + instance_class = None + _registered_providers = [] + + def __init__(self, handle = None): + if handle is not None: + self.handle = core.handle_of_type(handle, core.BNDownloadProvider) + self.__dict__["name"] = core.BNGetDownloadProviderName(handle) + + def register(self): + self._cb = core.BNDownloadProviderCallbacks() + self._cb.context = 0 + self._cb.createInstance = self._cb.createInstance.__class__(self._create_instance) + self.handle = core.BNRegisterDownloadProvider(self.__class__.name, self._cb) + self.__class__._registered_providers.append(self) + + def _create_instance(self, ctxt): + try: + result = self.__class__.instance_class(self) + if result is None: + return None + return ctypes.cast(core.BNNewDownloadInstanceReference(result.handle), ctypes.c_void_p).value + except: + log.log_error(traceback.format_exc()) + return None + + def create_instance(self): + result = core.BNCreateDownloadProviderInstance(self.handle) + if result is None: + return None + return DownloadInstance(self, handle = result) + + +class PythonDownloadInstance(DownloadInstance): + def __init__(self, provider): + super(PythonDownloadInstance, self).__init__(provider) + + @abc.abstractmethod + def perform_destroy_instance(self): + pass + + @abc.abstractmethod + def perform_request(self, ctxt, url): + try: + proxy_setting = Setting('download-client').get_string('https-proxy') + if proxy_setting: + opener = build_opener(ProxyHandler({'https': proxy_setting})) + install_opener(opener) + + r = urlopen(url) + total_size = int(r.headers.get('content-length', 0)) + bytes_sent = 0 + while True: + data = r.read(4096) + if not data: + break + raw_bytes = (ctypes.c_ubyte * len(data)).from_buffer_copy(data) + bytes_wrote = core.BNWriteDataForDownloadInstance(self.handle, raw_bytes, len(raw_bytes)) + if bytes_wrote != len(raw_bytes): + core.BNSetErrorForDownloadInstance(self.handle, "Bytes written mismatch!") + return -1 + bytes_sent = bytes_sent + bytes_wrote + continue_download = core.BNNotifyProgressForDownloadInstance(self.handle, bytes_sent, total_size) + if continue_download is False: + core.BNSetErrorForDownloadInstance(self.handle, "Download aborted!") + return -1 + + if not bytes_sent: + core.BNSetErrorForDownloadInstance(self.handle, "Received no data!") + return -1 + + except URLError as e: + core.BNSetErrorForDownloadInstance(self.handle, e.__class__.__name__) + return -1 + except: + core.BNSetErrorForDownloadInstance(self.handle, "Unknown Exception!") + log.log_error(traceback.format_exc()) + return -1 + + return 0 + + +class PythonDownloadProvider(DownloadProvider): + name = "DefaultDownloadProvider" + instance_class = PythonDownloadInstance + + +PythonDownloadProvider().register() -- cgit v1.3.1 From 67209107fbfe586a39101b7135b3f041cb934abe Mon Sep 17 00:00:00 2001 From: Ryan Snyder Date: Mon, 2 Jul 2018 12:35:24 -0400 Subject: downloadprovider: Fail gracefully on Ubuntu 14.04 installs, add fallback Provide a fallback so that installs with Python < 2.7.9 can still receive updates. Some manual intervention is required, but now we can provide working networking on any install that has either Python >= 2.7.9 or where a user can supply the requests[security] pip package --- python/downloadprovider.py | 176 +++++++++++++++++++++++++++++---------------- 1 file changed, 116 insertions(+), 60 deletions(-) (limited to 'python/downloadprovider.py') diff --git a/python/downloadprovider.py b/python/downloadprovider.py index 4756fd90..c0cecbe5 100644 --- a/python/downloadprovider.py +++ b/python/downloadprovider.py @@ -25,20 +25,11 @@ import ctypes import sys import traceback -try: - from urllib.parse import urlparse, urlencode - from urllib.request import urlopen, Request, build_opener, install_opener, ProxyHandler - from urllib.error import HTTPError, URLError -except ImportError: - from urlparse import urlparse - from urllib import urlencode - from urllib2 import urlopen, Request, build_opener, install_opener, ProxyHandler, HTTPError, URLError - # Binary Ninja Components -import _binaryninjacore as core +import binaryninja._binaryninjacore as core from binaryninja.setting import Setting -import startup -import log +from binaryninja import startup +from binaryninja import log class DownloadInstance(object): @@ -64,7 +55,7 @@ class DownloadInstance(object): def _perform_request(self, ctxt, url): try: - return self.perform_request(ctxt, url) + return self.perform_request(url) except: log.log_error(traceback.format_exc()) return -1 @@ -153,58 +144,123 @@ class DownloadProvider(object): return DownloadInstance(self, handle = result) -class PythonDownloadInstance(DownloadInstance): - def __init__(self, provider): - super(PythonDownloadInstance, self).__init__(provider) - - @abc.abstractmethod - def perform_destroy_instance(self): - pass - - @abc.abstractmethod - def perform_request(self, ctxt, url): - try: - proxy_setting = Setting('download-client').get_string('https-proxy') - if proxy_setting: - opener = build_opener(ProxyHandler({'https': proxy_setting})) - install_opener(opener) - - r = urlopen(url) - total_size = int(r.headers.get('content-length', 0)) - bytes_sent = 0 - while True: - data = r.read(4096) - if not data: - break - raw_bytes = (ctypes.c_ubyte * len(data)).from_buffer_copy(data) - bytes_wrote = core.BNWriteDataForDownloadInstance(self.handle, raw_bytes, len(raw_bytes)) - if bytes_wrote != len(raw_bytes): - core.BNSetErrorForDownloadInstance(self.handle, "Bytes written mismatch!") - return -1 - bytes_sent = bytes_sent + bytes_wrote - continue_download = core.BNNotifyProgressForDownloadInstance(self.handle, bytes_sent, total_size) - if continue_download is False: - core.BNSetErrorForDownloadInstance(self.handle, "Download aborted!") +if sys.version_info >= (2, 7, 9): + try: + from urllib.request import urlopen, build_opener, install_opener, ProxyHandler + from urllib.error import URLError + except ImportError: + from urllib2 import urlopen, build_opener, install_opener, ProxyHandler, URLError + + class PythonDownloadInstance(DownloadInstance): + def __init__(self, provider): + super(PythonDownloadInstance, self).__init__(provider) + + @abc.abstractmethod + def perform_destroy_instance(self): + pass + + @abc.abstractmethod + def perform_request(self, url): + try: + proxy_setting = Setting('download-client').get_string('https-proxy') + if proxy_setting: + opener = build_opener(ProxyHandler({'https': proxy_setting})) + install_opener(opener) + + r = urlopen(url.decode("charmap")) + total_size = int(r.headers.get('content-length', 0)) + bytes_sent = 0 + while True: + data = r.read(4096) + if not data: + break + raw_bytes = (ctypes.c_ubyte * len(data)).from_buffer_copy(data) + bytes_wrote = core.BNWriteDataForDownloadInstance(self.handle, raw_bytes, len(raw_bytes)) + if bytes_wrote != len(raw_bytes): + core.BNSetErrorForDownloadInstance(self.handle, "Bytes written mismatch!") + return -1 + bytes_sent = bytes_sent + bytes_wrote + continue_download = core.BNNotifyProgressForDownloadInstance(self.handle, bytes_sent, total_size) + if continue_download is False: + core.BNSetErrorForDownloadInstance(self.handle, "Download aborted!") + return -1 + + if not bytes_sent: + core.BNSetErrorForDownloadInstance(self.handle, "Received no data!") return -1 - if not bytes_sent: - core.BNSetErrorForDownloadInstance(self.handle, "Received no data!") + except URLError as e: + core.BNSetErrorForDownloadInstance(self.handle, e.__class__.__name__) + return -1 + except: + core.BNSetErrorForDownloadInstance(self.handle, "Unknown Exception!") + log.log_error(traceback.format_exc()) return -1 - except URLError as e: - core.BNSetErrorForDownloadInstance(self.handle, e.__class__.__name__) - return -1 - except: - core.BNSetErrorForDownloadInstance(self.handle, "Unknown Exception!") - log.log_error(traceback.format_exc()) - return -1 - - return 0 + return 0 + + class PythonDownloadProvider(DownloadProvider): + name = "DefaultDownloadProvider" + instance_class = PythonDownloadInstance + + PythonDownloadProvider().register() +else: + try: + import requests + from requests import pyopenssl + class PythonDownloadInstance(DownloadInstance): + def __init__(self, provider): + super(PythonDownloadInstance, self).__init__(provider) + + @abc.abstractmethod + def perform_destroy_instance(self): + pass + + @abc.abstractmethod + def perform_request(self, url): + try: + proxy_setting = Setting('download-client').get_string('https-proxy') + if proxy_setting: + proxies = {"https": proxy_setting} + else: + proxies = None + + r = requests.get(url.decode("charmap"), proxies=proxies) + if not r.ok: + core.BNSetErrorForDownloadInstance(self.handle, "Received error from server") + return -1 + data = r.content + if len(data) == 0: + core.BNSetErrorForDownloadInstance(self.handle, "No data received from server!") + return -1 + raw_bytes = (ctypes.c_ubyte * len(data)).from_buffer_copy(data) + bytes_wrote = core.BNWriteDataForDownloadInstance(self.handle, raw_bytes, len(raw_bytes)) + if bytes_wrote != len(raw_bytes): + core.BNSetErrorForDownloadInstance(self.handle, "Bytes written mismatch!") + return -1 + continue_download = core.BNNotifyProgressForDownloadInstance(self.handle, bytes_wrote, bytes_wrote) + if continue_download is False: + core.BNSetErrorForDownloadInstance(self.handle, "Download aborted!") + return -1 + except requests.RequestException as e: + core.BNSetErrorForDownloadInstance(self.handle, e.__class__.__name__) + return -1 + except: + core.BNSetErrorForDownloadInstance(self.handle, "Unknown Exception!") + log.log_error(traceback.format_exc()) + return -1 + return 0 -class PythonDownloadProvider(DownloadProvider): - name = "DefaultDownloadProvider" - instance_class = PythonDownloadInstance + class PythonDownloadProvider(DownloadProvider): + name = "DefaultDownloadProvider" + instance_class = PythonDownloadInstance + PythonDownloadProvider().register() + except ImportError: + log.log_error("On this version of Python, the pip requests[security] package is required for network connectivity!") + log.log_error("On an Ubuntu 14.04 install, the following three commands are sufficient to enable networking for the current user:") + log.log_error(" sudo apt install python-pip") + log.log_error(" python -m pip install pip --upgrade --user") + log.log_error(" python -m pip install requests[security] --upgrade --user") -PythonDownloadProvider().register() -- cgit v1.3.1 From 4f1a7824357ab58d868dbd675af9b152fc3694e9 Mon Sep 17 00:00:00 2001 From: Ryan Snyder Date: Fri, 6 Jul 2018 21:15:02 -0400 Subject: Python: More useful error when loading too-old Python on Windows --- python/downloadprovider.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'python/downloadprovider.py') diff --git a/python/downloadprovider.py b/python/downloadprovider.py index c0cecbe5..d34ca358 100644 --- a/python/downloadprovider.py +++ b/python/downloadprovider.py @@ -258,9 +258,12 @@ else: PythonDownloadProvider().register() except ImportError: - log.log_error("On this version of Python, the pip requests[security] package is required for network connectivity!") - log.log_error("On an Ubuntu 14.04 install, the following three commands are sufficient to enable networking for the current user:") - log.log_error(" sudo apt install python-pip") - log.log_error(" python -m pip install pip --upgrade --user") - log.log_error(" python -m pip install requests[security] --upgrade --user") + if sys.platform == "win32": + log.log_error("Provided Python version is too old! Only Python 2.7.10 and above are known to work on Windows!") + else: + log.log_error("On Python versions below 2.7.9, the pip requests[security] package is required for network connectivity!") + log.log_error("On an Ubuntu 14.04 install, the following three commands are sufficient to enable networking for the current user:") + log.log_error(" sudo apt install python-pip") + log.log_error(" python -m pip install pip --upgrade --user") + log.log_error(" python -m pip install requests[security] --upgrade --user") -- cgit v1.3.1