From 3d3b803d18f0d61b5e36367c9eb660b234cdc66e Mon Sep 17 00:00:00 2001 From: Peter LaFosse Date: Wed, 12 Jul 2017 16:04:28 -0400 Subject: Metadata enhancements. Metadata objects are now serialized to the DB --- python/metadata.py | 248 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 248 insertions(+) create mode 100644 python/metadata.py (limited to 'python/metadata.py') diff --git a/python/metadata.py b/python/metadata.py new file mode 100644 index 00000000..f0e7764d --- /dev/null +++ b/python/metadata.py @@ -0,0 +1,248 @@ +# Copyright (c) 2015-2017 Vector 35 LLC +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +import ctypes + +# Binary Ninja components +import _binaryninjacore as core +from enums import MetadataType + + +class Metadata(object): + def __init__(self, value=None, signed=None, raw=None, handle=None): + if handle is not None: + self.handle = handle + elif isinstance(value, int): + if signed: + self.handle = core.BNCreateMetadataSignedIntegerData(value) + else: + self.handle = core.BNCreateMetadataUnsignedIntegerData(value) + elif isinstance(value, bool): + self.handle = core.BNCreateMetadataBooleanData(value) + elif isinstance(value, str): + if raw: + buffer = (ctypes.c_ubyte * len(value)).from_buffer_copy(value) + self.handle = core.BNCreateMetadataRawData(buffer, len(value)) + else: + self.handle = core.BNCreateMetadataStringData(value) + elif isinstance(value, float): + self.handle = core.BNCreateMetadataDoubleData(value) + elif isinstance(value, list): + self.handle = core.BNCreateMetadataOfType(MetadataType.ArrayDataType) + for elm in value: + md = Metadata(elm, signed, raw) + core.BNMetadataArrayAppend(self.handle, md.handle) + elif isinstance(value, dict): + self.handle = core.BNCreateMetadataOfType(MetadataType.KeyValueDataType) + for elm in value: + md = Metadata(value[elm], signed, raw) + core.BNMetadataSetValueForKey(self.handle, str(elm), md.handle) + else: + raise ValueError("List doesn't not contain type of: int, bool, str, float, list, dict") + + @property + def value(self): + if self.is_integer: + return int(self) + elif self.is_string or self.is_raw: + return str(self) + elif self.is_float: + return float(self) + elif self.is_boolean: + return bool(self) + elif self.is_array: + return list(self) + elif self.is_dict: + return dict(self) + raise NotImplementedError() + + @property + def type(self): + return MetadataType(core.BNMetadataGetType(self.handle)) + + @property + def is_integer(self): + return self.is_signed_integer or self.is_unsigned_integer + + @property + def is_signed_integer(self): + return core.BNMetadataIsSignedInteger(self.handle) + + @property + def is_unsigned_integer(self): + return core.BNMetadataIsUnsignedInteger(self.handle) + + @property + def is_float(self): + return core.BNMetadataIsDouble(self.handle) + + @property + def is_boolean(self): + return core.BNMetadataIsBoolean(self.handle) + + @property + def is_string(self): + return core.BNMetadataIsString(self.handle) + + @property + def is_raw(self): + return core.BNMetadataIsRaw(self.handle) + + @property + def is_array(self): + return core.BNMetadataIsArray(self.handle) + + @property + def is_dict(self): + return core.BNMetadataIsKeyValueStore(self.handle) + + def __len__(self): + if self.is_array or self.is_dict or self.is_string or self.is_raw: + return core.BNMetadataSize(self.handle) + raise Exception("Metadata object doesn't support len()") + + def __iter__(self): + if self.is_array: + for i in xrange(core.BNMetadataSize(self.handle)): + yield Metadata(handle=core.BNMetadataGetForIdx(self.handle, i)) + elif self.is_dict: + result = core.BNMetadataGetValueStore(self.handle) + try: + for i in xrange(result.contents.size): + yield result.contents.keys[i] + finally: + core.BNFreeMetadataValueStore(result) + else: + raise Exception("Metadata object doesn't support iteration") + + def __getitem__(self, value): + if self.is_array: + if not isinstance(value, int): + raise ValueError("Metadata object only supports integers for indexing") + if value >= len(self): + raise IndexError("Index value out of range") + return Metadata(handle=core.BNMetadataGetForIdx(self.handle, value)) + if self.is_dict: + if not isinstance(value, str): + raise ValueError("Metadata object only supports strings for indexing") + handle = core.BNMetadataGetForKey(self.handle, value) + if handle is None: + raise KeyError(value) + return Metadata(handle=handle) + + def __str__(self): + if self.is_string: + return core.BNMetadataGetString(self.handle) + if self.is_raw: + length = ctypes.c_ulonglong() + length.value = 0 + native_list = core.BNMetadataGetRaw(self.handle, ctypes.byref(length)) + out_list = [] + for i in xrange(length.value): + out_list.append(native_list[i]) + core.BNFreeMetadataRaw(native_list) + return ''.join(chr(a) for a in out_list) + + raise ValueError("Metadata object not a string or raw type") + + def __int__(self): + if self.is_signed_integer: + return core.BNMetadataGetSignedInteger(self.handle) + if self.is_unsigned_integer: + return core.BNMetadataGetUnsignedInteger(self.handle) + + raise ValueError("Metadata object not of integer type") + + def __float__(self): + if not self.is_float: + raise ValueError("Metadata object is not float type") + return core.BNMetadataGetDouble(self.handle) + + def __nonzero__(self): + if not self.is_boolean: + raise ValueError("Metadata object is not boolean type") + return core.BNMetadataGetBoolean(self.handle) + + def __eq__(self, other): + if isinstance(other, int) and self.is_integer: + return int(self) == other + elif isinstance(other, str) and (self.is_string or self.is_raw): + return str(self) == other + elif isinstance(other, float) and self.is_float: + return float(self) == other + elif isinstance(other, bool) and self.is_boolean: + return bool(self) == other + elif self.is_array and ((isinstance(other, Metadata) and other.is_array) or isinstance(other, list)): + if len(self) != len(other): + return False + for a, b in zip(self, other): + if a != b: + return False + return True + elif self.is_dict and ((isinstance(other, Metadata) and other.is_dict) or isinstance(other, dict)): + if len(self) != len(other): + return False + for a, b in zip(self, other): + if a != b or self[a] != other[b]: + return False + return True + elif isinstance(other, Metadata) and self.is_integer and other.is_integer: + return int(self) == int(other) + elif isinstance(other, Metadata) and (self.is_string or self.is_raw) and (other.is_string or other.is_raw): + return str(self) == str(other) + elif isinstance(other, Metadata) and self.is_float and other.is_float: + return float(self) == float(other) + elif isinstance(other, Metadata) and self.is_boolean and other.is_boolean: + return bool(self) == bool(other) + raise NotImplementedError() + + def __ne__(self, other): + if isinstance(other, int) and self.is_integer: + return int(self) != other + elif isinstance(other, str) and (self.is_string or self.is_raw): + return str(self) != other + elif isinstance(other, float) and self.is_float: + return float(self) != other + elif isinstance(other, bool): + return bool(self) != other + elif self.is_array and ((isinstance(other, Metadata) and other.is_array) or isinstance(other, list)): + if len(self) != len(other): + return True + areEqual = True + for a, b in zip(self, other): + if a != b: + areEqual = False + return not areEqual + elif self.is_dict and ((isinstance(other, Metadata) and other.is_dict) or isinstance(other, dict)): + if len(self) != len(other): + return True + for a, b in zip(self, other): + if a != b or self[a] != other[b]: + return True + return False + elif isinstance(other, Metadata) and self.is_integer and other.is_integer: + return int(self) != int(other) + elif isinstance(other, Metadata) and (self.is_string or self.is_raw) and (other.is_string or other.is_raw): + return str(self) != str(other) + elif isinstance(other, Metadata) and self.is_float and other.is_float: + return float(self) != float(other) + elif isinstance(other, Metadata) and self.is_boolean and other.is_boolean: + return bool(self) != bool(other) -- cgit v1.3.1 From 0e6019edc8c5949de4989ef9577c07913946135b Mon Sep 17 00:00:00 2001 From: Peter LaFosse Date: Wed, 12 Jul 2017 22:47:12 -0400 Subject: Adding remove_metadata API to BinaryView. Add remove APIs to Metadata --- binaryninjaapi.h | 6 ++++-- binaryninjacore.h | 6 +++++- binaryview.cpp | 5 +++++ metadata.cpp | 12 +++++++++++- python/binaryview.py | 15 ++++++++++++++- python/metadata.py | 12 ++++++++++-- 6 files changed, 49 insertions(+), 7 deletions(-) (limited to 'python/metadata.py') diff --git a/binaryninjaapi.h b/binaryninjaapi.h index 83f6b582..01e5f986 100644 --- a/binaryninjaapi.h +++ b/binaryninjaapi.h @@ -1130,6 +1130,7 @@ namespace BinaryNinja void StoreMetadata(const std::string& key, Ref value); Ref QueryMetadata(const std::string& key); + void RemoveMetadata(const std::string& key); std::string GetStringMetadata(const std::string& key); std::vector GetRawMetadata(const std::string& key); uint64_t GetUIntMetadata(const std::string& key); @@ -2956,11 +2957,12 @@ namespace BinaryNinja //For key-value data only Ref Get(const std::string& key); bool SetValueForKey(const std::string& key, Ref data); + void RemoveKey(const std::string& key); //For array data only - Ref Get(size_t idx); + Ref Get(size_t index); bool Append(Ref data); - + void RemoveIndex(size_t index); size_t Size() const; bool IsBoolean() const; diff --git a/binaryninjacore.h b/binaryninjacore.h index 54691360..3a8b63df 100644 --- a/binaryninjacore.h +++ b/binaryninjacore.h @@ -2939,8 +2939,10 @@ extern "C" BINARYNINJACOREAPI bool BNMetadataSetValueForKey(BNMetadata* data, const char* key, BNMetadata* md); BINARYNINJACOREAPI BNMetadata* BNMetadataGetForKey(BNMetadata* data, const char* key); BINARYNINJACOREAPI bool BNMetadataArrayAppend(BNMetadata* data, BNMetadata* md); + BINARYNINJACOREAPI void BNMetadataRemoveKey(BNMetadata* data, const char* key); BINARYNINJACOREAPI size_t BNMetadataSize(BNMetadata* data); - BINARYNINJACOREAPI BNMetadata* BNMetadataGetForIdx(BNMetadata* data, size_t idx); + BINARYNINJACOREAPI BNMetadata* BNMetadataGetForIndex(BNMetadata* data, size_t index); + BINARYNINJACOREAPI void BNMetadataRemoveIndex(BNMetadata* data, size_t index); BINARYNINJACOREAPI void BNFreeMetadataArray(BNMetadata** data); BINARYNINJACOREAPI void BNFreeMetadataValueStore(BNMetadataValueStore* data); @@ -2970,6 +2972,8 @@ extern "C" // Store/Query structured data to/from a BinaryView BINARYNINJACOREAPI void BNBinaryViewStoreMetadata(BNBinaryView* view, const char* key, BNMetadata* value); BINARYNINJACOREAPI BNMetadata* BNBinaryViewQueryMetadata(BNBinaryView* view, const char* key); + BINARYNINJACOREAPI void BNBinaryViewRemoveMetadata(BNBinaryView* view, const char* key); + #ifdef __cplusplus } #endif diff --git a/binaryview.cpp b/binaryview.cpp index 3f1196d0..1f6820ff 100644 --- a/binaryview.cpp +++ b/binaryview.cpp @@ -1842,6 +1842,11 @@ Ref BinaryView::QueryMetadata(const std::string& key) return new Metadata(value); } +void BinaryView::RemoveMetadata(const std::string& key) +{ + BNBinaryViewRemoveMetadata(m_object, key.c_str()); +} + string BinaryView::GetStringMetadata(const string& key) { auto data = QueryMetadata(key); diff --git a/metadata.cpp b/metadata.cpp index 18d3500b..f9c48b04 100644 --- a/metadata.cpp +++ b/metadata.cpp @@ -87,7 +87,7 @@ Ref Metadata::operator[](const std::string& key) Ref Metadata::operator[](size_t idx) { - return new Metadata(BNMetadataGetForIdx(m_object, idx)); + return new Metadata(BNMetadataGetForIndex(m_object, idx)); } bool Metadata::SetValueForKey(const string& key, Ref data) @@ -95,6 +95,11 @@ bool Metadata::SetValueForKey(const string& key, Ref data) return BNMetadataSetValueForKey(m_object, key.c_str(), data->m_object); } +void Metadata::RemoveKey(const string& key) +{ + return BNMetadataRemoveKey(m_object, key.c_str()); +} + MetadataType Metadata::GetType() const { return BNMetadataGetType(m_object); @@ -160,6 +165,11 @@ bool Metadata::Append(Ref data) return BNMetadataArrayAppend(m_object, data->m_object); } +void Metadata::RemoveIndex(size_t index) +{ + BNMetadataRemoveIndex(m_object, index); +} + size_t Metadata::Size() const { return BNMetadataSize(m_object); diff --git a/python/binaryview.py b/python/binaryview.py index d04dca86..ee019dcb 100644 --- a/python/binaryview.py +++ b/python/binaryview.py @@ -27,7 +27,7 @@ import threading # Binary Ninja components import _binaryninjacore as core from enums import (AnalysisState, SymbolType, InstructionTextTokenType, - Endianness, ModificationStatus, StringType, SegmentFlag, MetadataType) + Endianness, ModificationStatus, StringType, SegmentFlag) import function import startup import architecture @@ -3312,6 +3312,19 @@ class BinaryView(object): raise ValueError("metadata argument must be of type Metadata") core.BNBinaryViewStoreMetadata(self.handle, key, md.handle) + def remove_metadata(self, key): + """ + `remove_metadata` removes the Metadata object associated with key from the current BinaryView + + :param string key: key to remove from the BinaryView + :rtype: None + :Example: + + >>> bv.store_metadata("integer", Metadata(1337)) + >>> bv.remove_metadata("integer") + """ + core.BNBinaryViewRemoveMetadata(self.handle, key) + def __setattr__(self, name, value): try: object.__setattr__(self, name, value) diff --git a/python/metadata.py b/python/metadata.py index f0e7764d..2817d777 100644 --- a/python/metadata.py +++ b/python/metadata.py @@ -114,6 +114,14 @@ class Metadata(object): def is_dict(self): return core.BNMetadataIsKeyValueStore(self.handle) + def remove(self, key_or_index): + if isinstance(key_or_index, str) and self.is_dict: + core.BNMetadataRemoveKey(self.handle, key_or_index) + elif isinstance(key_or_index, int) and self.is_array: + core.BNMetadataRemoveIndex(self.handle, key_or_index) + else: + raise TypeError("remove only valid for dict and array objects") + def __len__(self): if self.is_array or self.is_dict or self.is_string or self.is_raw: return core.BNMetadataSize(self.handle) @@ -122,7 +130,7 @@ class Metadata(object): def __iter__(self): if self.is_array: for i in xrange(core.BNMetadataSize(self.handle)): - yield Metadata(handle=core.BNMetadataGetForIdx(self.handle, i)) + yield Metadata(handle=core.BNMetadataGetForIndex(self.handle, i)) elif self.is_dict: result = core.BNMetadataGetValueStore(self.handle) try: @@ -139,7 +147,7 @@ class Metadata(object): raise ValueError("Metadata object only supports integers for indexing") if value >= len(self): raise IndexError("Index value out of range") - return Metadata(handle=core.BNMetadataGetForIdx(self.handle, value)) + return Metadata(handle=core.BNMetadataGetForIndex(self.handle, value)) if self.is_dict: if not isinstance(value, str): raise ValueError("Metadata object only supports strings for indexing") -- cgit v1.3.1 From 755a1a40c636f5fbd463ffdda646d92ae68ebada Mon Sep 17 00:00:00 2001 From: Peter LaFosse Date: Thu, 13 Jul 2017 18:07:20 -0400 Subject: Make query/store metadata completely duck typed --- python/binaryview.py | 50 +++++++++++++++++++++++++------------------------- python/metadata.py | 20 +++++++++++++++----- 2 files changed, 40 insertions(+), 30 deletions(-) (limited to 'python/metadata.py') diff --git a/python/binaryview.py b/python/binaryview.py index ee019dcb..82c9b727 100644 --- a/python/binaryview.py +++ b/python/binaryview.py @@ -3266,61 +3266,61 @@ class BinaryView(object): def query_metadata(self, key): """ - `query_metadata` retrieves a Metadata object stored in the current BinaryView. + `query_metadata` retrieves a metadata associated with the given key stored in the current BinaryView. :param string key: key to query - :rtype: Metadata object + :rtype: metadata associated with the key :Example: - >>> bv.store_metadata("integer", Metadata(1337)) - >>> int(bv.query_metadata("integer")) + >>> bv.store_metadata("integer", 1337) + >>> bv.query_metadata("integer") 1337L - >>> bv.store_metadata("list", Metadata([1,2,3])) - >>> map(int, list(bv.query_metadata("list"))) + >>> bv.store_metadata("list", [1,2,3]) + >>> bv.query_metadata("list") [1L, 2L, 3L] - >>> bv.store_metadata("string", Metadata("my_data")) - >>> str(bv.query_metadata("string")) + >>> bv.store_metadata("string", "my_data") + >>> bv.query_metadata("string") 'my_data' """ md_handle = core.BNBinaryViewQueryMetadata(self.handle, key) if md_handle is None: raise KeyError(key) - return metadata.Metadata(handle=md_handle) + return metadata.Metadata(handle=md_handle).value def store_metadata(self, key, md): """ - `store_metadata` stores a Metadata object for the given key in the current BinaryView. - Metadata objects stored using this `store_metadata` are stored in the database and can be retrieved when - the database is reopend. + `store_metadata` stores an object for the given key in the current BinaryView. Objects stored using + `store_metadata` can be retrieved when the database is reopend. Objects stored are not arbitrary python + objects! The values stored must be able to be held in a Metadata object. See :py:class:`Metadata` + for more information. Python objects could obviously be serialized using pickle but this intentionally + a task left to the user since there is the potential security issues. :param string key: key value to associate the Metadata object with - :param Metadata md: Metadata object to store + :param Varies md: object to store. :rtype: None :Example: - >>> bv.store_metadata("integer", Metadata(1337)) - >>> int(bv.query_metadata("integer")) + >>> bv.store_metadata("integer", 1337) + >>> bv.query_metadata("integer") 1337L - >>> bv.store_metadata("list", Metadata([1,2,3])) - >>> map(int, list(bv.query_metadata("list"))) + >>> bv.store_metadata("list", [1,2,3]) + >>> bv.query_metadata("list") [1L, 2L, 3L] - >>> bv.store_metadata("string", Metadata("my_data")) - >>> str(bv.query_metadata("string")) + >>> bv.store_metadata("string", "my_data") + >>> bv.query_metadata("string") 'my_data' """ - if not isinstance(md, metadata.Metadata): - raise ValueError("metadata argument must be of type Metadata") - core.BNBinaryViewStoreMetadata(self.handle, key, md.handle) + core.BNBinaryViewStoreMetadata(self.handle, key, metadata.Metadata(md).handle) def remove_metadata(self, key): """ - `remove_metadata` removes the Metadata object associated with key from the current BinaryView + `remove_metadata` removes the metadata associated with key from the current BinaryView. - :param string key: key to remove from the BinaryView + :param string key: key associated with metadata to remove from the BinaryView :rtype: None :Example: - >>> bv.store_metadata("integer", Metadata(1337)) + >>> bv.store_metadata("integer", 1337) >>> bv.remove_metadata("integer") """ core.BNBinaryViewRemoveMetadata(self.handle, key) diff --git a/python/metadata.py b/python/metadata.py index 2817d777..554bbcf4 100644 --- a/python/metadata.py +++ b/python/metadata.py @@ -71,8 +71,16 @@ class Metadata(object): elif self.is_array: return list(self) elif self.is_dict: - return dict(self) - raise NotImplementedError() + return self.get_dict() + raise TypeError() + + def get_dict(self): + if not self.is_dict: + raise TypeError() + result = {} + for key in self: + result[key] = self[key] + return result @property def type(self): @@ -130,7 +138,7 @@ class Metadata(object): def __iter__(self): if self.is_array: for i in xrange(core.BNMetadataSize(self.handle)): - yield Metadata(handle=core.BNMetadataGetForIndex(self.handle, i)) + yield Metadata(handle=core.BNMetadataGetForIndex(self.handle, i)).value elif self.is_dict: result = core.BNMetadataGetValueStore(self.handle) try: @@ -147,14 +155,16 @@ class Metadata(object): raise ValueError("Metadata object only supports integers for indexing") if value >= len(self): raise IndexError("Index value out of range") - return Metadata(handle=core.BNMetadataGetForIndex(self.handle, value)) + return Metadata(handle=core.BNMetadataGetForIndex(self.handle, value)).value if self.is_dict: if not isinstance(value, str): raise ValueError("Metadata object only supports strings for indexing") handle = core.BNMetadataGetForKey(self.handle, value) if handle is None: raise KeyError(value) - return Metadata(handle=handle) + return Metadata(handle=handle).value + + raise NotImplementedError("Metadata object doesn't support indexing") def __str__(self): if self.is_string: -- cgit v1.3.1