tests: Separate unit test corpus from results

6 files changed, 90 insertions, 457 deletions

diff --git a/python/binaryview.py b/python/binaryview.py
index d439f563..cdc88fff 100644
--- a/python/binaryview.py
+++ b/python/binaryview.py
@@ -272,7 +272,7 @@ class BinaryDataNotificationCallbacks(object):
 
 	def _function_update_requested(self, ctxt, view, func):
 		try:
-			self.notify.function_update_requested(self.view, function.Function(self.view, core.BNNewFunctionReference(func)))
+			self.notify.function_update_requested(self.view, binaryninja.function.Function(self.view, core.BNNewFunctionReference(func)))
 		except:
 			log.log_error(traceback.format_exc())
 
diff --git a/suite/api_test.py b/suite/api_test.py
index f6468941..740f88e3 100644
--- a/suite/api_test.py
+++ b/suite/api_test.py
@@ -1,17 +1,10 @@
 import unittest
-import time
 import platform
 import os
-import tempfile
-import zipfile
-import urllib
-import subprocess
-import getpass
 from binaryninja.setting import Setting
 from binaryninja.metadata import Metadata
 from binaryninja.demangle import demangle_gnu3, get_qualified_name
 from binaryninja.architecture import Architecture
-from binaryninja.binaryview import BinaryViewType
 
 
 class SettingsAPI(unittest.TestCase):
@@ -226,41 +219,3 @@ class DemanglerTest(unittest.TestCase):
 		for i, test in enumerate(tests):
 			t, n = demangle_gnu3(Architecture['x86'], test)
 			assert self.get_type_string(t, n) == results[i]
-
-
-class TimingTest(unittest.TestCase):
-	def unpackage_file(self, file):
-		if not os.path.exists(file):
-			with zipfile.ZipFile(file + ".zip", "r") as zf:
-				zf.extractall()
-		assert os.path.exists(file)
-
-	def test_analysis_time(self):
-		if platform.system() != "Linux" or getpass.getuser() != "jenkins":
-			return
-
-		start_time = time.time()
-		file_names = [os.path.join(os.path.dirname(__file__), "binaries", "quick3dcoreplugin.dll"),
-					os.path.join(os.path.dirname(__file__), "binaries", "md5"),
-					os.path.join(os.path.dirname(__file__), "binaries", "ls")]
-
-		for file_name in file_names:
-			temp_name = next(tempfile._get_candidate_names()) + ".bndb"
-			self.unpackage_file(file_name)
-			try:
-				bv = BinaryViewType.get_view_of_file(file_name)
-				bv.create_database(temp_name)
-				bv.file.close()
-				bv = BinaryViewType.get_view_of_file(temp_name)
-				bv.file.close()
-			finally:
-				if os.path.exists(file_name):
-					os.unlink(file_name)
-				if os.path.exists(temp_name):
-					os.unlink(temp_name)
-
-		log_url = os.environ.get('BINJA_LOG_URL')
-		if log_url:
-			time_s = time.time() - start_time
-			commit = subprocess.check_output(["git", "rev-parse", "HEAD"])[:-1]
-			conn = urllib.urlopen("%s?BuildID=%s&Test1=%.2f" % (log_url, commit, time_s))
diff --git a/suite/binaries b/suite/binaries
-Subproject d1f1e6f7e2207338a9f92d5980a2a795d1aca1c
+Subproject 96d08bbb4fdcf1b88c98165fcfe8474659c38d6
diff --git a/suite/generator.py b/suite/generator.py
index f4ee69ee..09ee00dc 100755
--- a/suite/generator.py
+++ b/suite/generator.py
@@ -14,11 +14,14 @@ import sys
 import unittest
 import pickle
 import zipfile
-import testcommon
-import api_test
 import difflib
 from collections import Counter
 
+api_suite_path = os.path.join(os.path.dirname(__file__), {4})
+sys.path.append(api_suite_path)
+import testcommon
+import api_test
+
 global verbose
 verbose = False
 
@@ -77,27 +80,29 @@ class TestBinaryNinjaAPI(unittest.TestCase):
     @classmethod
     def setUpClass(self):
         self.builder = testcommon.TestBuilder("{3}")
+        pickle_path = os.path.join(os.path.dirname(__file__), "oracle.pkl")
         try:
             # Python 2 does not have the encodings option
-            self.oracle_test_data = pickle.load(open(os.path.join("{0}", "oracle.pkl"), "rb"), encoding='charmap')
+            self.oracle_test_data = pickle.load(open(pickle_path, "rb"), encoding='charmap')
         except TypeError:
-            self.oracle_test_data = pickle.load(open(os.path.join("{0}", "oracle.pkl"), "rb"))
+            self.oracle_test_data = pickle.load(open(pickle_path, "rb"))
         self.verifybuilder = testcommon.VerifyBuilder("{3}")
 
     def run_binary_test(self, testfile):
         testname = None
-        with zipfile.ZipFile(testfile, "r") as zf:
+        with zipfile.ZipFile(os.path.join(api_suite_path, testfile), "r") as zf:
             testname = zf.namelist()[0]
-            zf.extractall()
+            zf.extractall(path=api_suite_path)
 
-        self.assertTrue(os.path.exists(testname + ".pkl"), "Test pickle doesn't exist")
+        pickle_path = os.path.join(os.path.dirname(__file__), testname + ".pkl")
+        self.assertTrue(pickle_path, "Test pickle doesn't exist")
         try:
             # Python 2 does not have the encodings option
-            binary_oracle = pickle.load(open(testname + ".pkl", "rb"), encoding='charmap')
+            binary_oracle = pickle.load(open(pickle_path, "rb"), encoding='charmap')
         except TypeError:
-            binary_oracle = pickle.load(open(testname + ".pkl", "rb"))
+            binary_oracle = pickle.load(open(pickle_path, "rb"))
 
-        test_builder = testcommon.BinaryViewTestBuilder(testname, "{3}")
+        test_builder = testcommon.BinaryViewTestBuilder(testname)
         for method in test_builder.methods():
             test = getattr(test_builder, method)()
             oracle = binary_oracle[method]
@@ -108,7 +113,7 @@ class TestBinaryNinjaAPI(unittest.TestCase):
             result += ":\\n"
             report = self.report(oracle, test, result)
             self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-        os.unlink(testname)
+        os.unlink(os.path.join(api_suite_path, testname))
 {1}{2}
 
 if __name__ == "__main__":
@@ -127,6 +132,7 @@ binary_test_string = """
     def test_binary__{0}(self):
         self.run_binary_test('{1}')
 """
+
 test_string = """
     def {0}(self):
         oracle = self.oracle_test_data['{0}']
@@ -167,7 +173,11 @@ class UnitTestFile:
         self.binary_tests = ""
 
     def close(self):
-        self.f.write(self.template.format(self.outdir, self.tests, self.binary_tests, self.test_store).encode('charmap'))
+        api_path = os.path.relpath(os.path.dirname(__file__), start=self.outdir)
+        api_path = os.path.normpath(api_path)
+        api_path = map(lambda x: '"{0}"'.format(x), api_path.split(os.sep))
+        api_path = '{0}'.format(', '.join(api_path))
+        self.f.write(self.template.format(self.outdir, self.tests, self.binary_tests, self.test_store, api_path).encode('charmap'))
         self.f.close()
 
     def add_verify(self, test_name):
@@ -209,7 +219,7 @@ class TestStoreError(Exception):
 
 
 def generate(test_store, outdir, exclude_binaries):
-    if not os.path.isdir(test_store):
+    if not os.path.isdir(os.path.join(os.path.dirname(__file__), test_store)):
         raise TestStoreError("Specified test store is not a directory")
 
     unittest = UnitTestFile(os.path.join(outdir, "unit.py"), outdir, test_store)
@@ -238,10 +248,12 @@ def generate(test_store, outdir, exclude_binaries):
         oraclefile = None
         if testfile.endswith(".pkl"):
             continue
+        elif testfile.endswith(".DS_Store"):
+            continue
         elif testfile.endswith(".zip"):
             # We have a zipped binary unzip it so we can rebaseline
             with zipfile.ZipFile(testfile, "r") as zf:
-                zf.extractall()
+                zf.extractall(path = os.path.dirname(__file__))
             if not os.path.exists(testfile[:-4]):
                 print("Error extracting testfile %s from zip: %s" % (testfile[:-4], testfile))
                 continue
@@ -253,14 +265,16 @@ def generate(test_store, outdir, exclude_binaries):
             # We have a binary that isn't zipped use it as a new test case
             oraclefile = testfile
 
+        oraclefile_rel = os.path.relpath(oraclefile, start=os.path.dirname(__file__))
+
         # Now generate the oracle data
-        update_progress(progress, len(allfiles), oraclefile)
-        unittest.add_binary_test(test_store, oraclefile)
+        update_progress(progress, len(allfiles), oraclefile_rel)
+        unittest.add_binary_test(test_store, oraclefile_rel)
         binary_start_time = time.time()
         if exclude_binaries:
             continue
-        test_data = testcommon.BinaryViewTestBuilder(oraclefile, test_store)
-        binary_oracle = OracleTestFile(oraclefile)
+        test_data = testcommon.BinaryViewTestBuilder(oraclefile_rel)
+        binary_oracle = OracleTestFile(os.path.join(outdir, oraclefile_rel))
         for method in test_data.methods():
             binary_oracle.add_entry(test_data, method)
         binary_oracle.close()
@@ -268,7 +282,7 @@ def generate(test_store, outdir, exclude_binaries):
 
         if not os.path.exists(oraclefile + ".zip"):
             with zipfile.ZipFile(oraclefile + ".zip", "w") as zf:
-                zf.write(oraclefile)
+                zf.write(oraclefile, os.path.relpath(oraclefile, start=os.path.dirname(__file__)))
 
         os.unlink(oraclefile)
 
@@ -288,17 +302,13 @@ def main():
                       default=False, help="Exclude regeneration of binaries")
     parser.add_option("-o", "--outputdir", default="suite",
                       dest="outputdir", action="store", type="string",
-                      help="output directory where the unit.py and oracle.py files will be stored")
-    parser.add_option("-i", "--inputdir", default=os.path.join("suite", "binaries", "test_corpus"),
+                      help="output directory where the unit.py and oracle.py files will be stored (relative to cwd)")
+    parser.add_option("-i", "--inputdir", default=os.path.join("binaries", "test_corpus"),
                       dest="test_store", action="store", type="string",
-                      help="input directory containing the binaries you which to generate unit tests from")
+                      help="input directory containing the binaries you which to generate unit tests from (relative to this file)")
 
     options, args = parser.parse_args()
 
-    if not os.path.exists(os.path.join(os.getcwd(), 'suite')):
-        print("Error: Please run this script from the binaryninja-api root directory")
-        sys.exit(1)
-
     myprint("[+] INFO: Using test store: %s" % options.test_store)
     if len(testcommon.get_file_list(options.test_store)) == 0:
         myprint("ERROR: No files in the test store %s" % testcommon.get_file_list(options.test_store))
diff --git a/suite/testcommon.py b/suite/testcommon.py
index 07ceb592..3cf981ee 100644
--- a/suite/testcommon.py
+++ b/suite/testcommon.py
@@ -51,7 +51,8 @@ def fixSet(string):
         return string
 
 
-def get_file_list(test_store):
+def get_file_list(test_store_rel):
+    test_store = os.path.join(os.path.dirname(__file__), test_store_rel)
     all_files = []
     for root, dir, files in os.walk(test_store):
         for file in files:
@@ -67,7 +68,6 @@ def remove_low_confidence(type_string):
 class Builder(object):
     def __init__(self, test_store):
         self.test_store = test_store
-        self.examples_dir = os.path.join(self.test_store, "..", "..", "..", "python", "examples")
 
     def methods(self):
         methodnames = []
@@ -76,11 +76,13 @@ class Builder(object):
                 methodnames.append(methodname)
         return methodnames
 
-    def unpackage_file(self, file):
-        if not os.path.exists(file):
-            with zipfile.ZipFile(file + ".zip", "r") as zf:
-                zf.extractall()
-        assert os.path.exists(file)
+    def unpackage_file(self, filename):
+        path = os.path.join(os.path.dirname(__file__), self.test_store, filename)
+        if not os.path.exists(path):
+            with zipfile.ZipFile(path + ".zip", "r") as zf:
+                zf.extractall(path = os.path.dirname(__file__))
+        assert os.path.exists(path)
+        return os.path.relpath(path)
 
 
 class BinaryViewTestBuilder(Builder):
@@ -92,9 +94,9 @@ class BinaryViewTestBuilder(Builder):
          - Function doc string used as 'on error' message
          - Should return: list of strings
     """
-    def __init__(self, filename, test_store):
-        self.filename = filename
-        self.bv = BinaryViewType.get_view_of_file(filename)
+    def __init__(self, filename):
+        self.filename = os.path.join(os.path.dirname(__file__), filename)
+        self.bv = BinaryViewType.get_view_of_file(self.filename)
         if self.bv is None:
             print("%s is not an executable format" % filename)
             return
@@ -539,7 +541,7 @@ class TestBuilder(Builder):
 
     def test_Types(self):
         """Types produced different result"""
-        file_name = os.path.join(self.test_store, "helloworld")
+        file_name = self.unpackage_file("helloworld")
         bv = binja.BinaryViewType.get_view_of_file(file_name)
 
         preprocessed = binja.preprocess_source("""
@@ -569,17 +571,16 @@ class TestBuilder(Builder):
 
     def test_Plugin_bin_info(self):
         """print_syscalls plugin produced different result"""
-        file_name = os.path.join(self.test_store, "helloworld")
-        self.unpackage_file(file_name)
-        result = subprocess.Popen(["python", os.path.join(self.examples_dir, "bin_info.py"), file_name], stdout=subprocess.PIPE).communicate()[0]
+        file_name = self.unpackage_file("helloworld")
+        bin_info_path = os.path.join(os.path.dirname(__file__), '..', 'python', 'examples', 'bin_info.py')
+        result = subprocess.Popen(["python", bin_info_path, file_name], stdout=subprocess.PIPE).communicate()[0]
         # normalize line endings and path sep
         return [line for line in result.replace(b"\\", b"/").replace(b"\r\n", b"\n").decode("charmap").split("\n")]
 
     def test_linear_disassembly(self):
         """linear_disassembly produced different result"""
-        file = os.path.join(self.test_store, "helloworld")
-        self.unpackage_file(file)
-        bv = binja.BinaryViewType['ELF'].open(file)
+        file_name = self.unpackage_file("helloworld")
+        bv = binja.BinaryViewType['ELF'].open(file_name)
         disass = bv.linear_disassembly
         retinfo = []
         for i in disass:
@@ -590,31 +591,26 @@ class TestBuilder(Builder):
 
     def test_partial_register_dataflow(self):
         """partial_register_dataflow produced different results"""
-        file_name = os.path.join(self.test_store, "partial_register_dataflow")
-        self.unpackage_file(file_name)
+        file_name = self.unpackage_file("partial_register_dataflow")
         result = []
-        try:
-            reg_list = ['ch', 'cl', 'ah', 'edi', 'al', 'cx', 'ebp', 'ax', 'edx', 'ebx', 'esp', 'esi', 'dl', 'dh', 'di', 'bl', 'bh', 'eax', 'dx', 'bx', 'ecx', 'sp', 'si']
-            bv = binja.BinaryViewType.get_view_of_file(file_name)
-            for func in bv.functions:
-                llil = func.low_level_il
-                for i in range(0, llil.__len__()-1):
-                    for x in reg_list:
-                        result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_reg_value(x)).replace('L', ''))
-                        result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_possible_reg_values(x)).replace('L', ''))
-                        result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_reg_value_after(x)).replace('L', ''))
-                        result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_possible_reg_values_after(x)).replace('L', ''))
-            bv.file.close()
-            del bv
-        finally:
-            os.unlink(file_name)
+        reg_list = ['ch', 'cl', 'ah', 'edi', 'al', 'cx', 'ebp', 'ax', 'edx', 'ebx', 'esp', 'esi', 'dl', 'dh', 'di', 'bl', 'bh', 'eax', 'dx', 'bx', 'ecx', 'sp', 'si']
+        bv = binja.BinaryViewType.get_view_of_file(file_name)
+        for func in bv.functions:
+            llil = func.low_level_il
+            for i in range(0, llil.__len__()-1):
+                for x in reg_list:
+                    result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_reg_value(x)).replace('L', ''))
+                    result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_possible_reg_values(x)).replace('L', ''))
+                    result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_reg_value_after(x)).replace('L', ''))
+                    result.append("LLIL:" + str(i).replace('L', '') + ":" + x + ":" + str(llil[i].get_possible_reg_values_after(x)).replace('L', ''))
+        bv.file.close()
+        del bv
         return result
 
 
     def test_low_il_stack(self):
         """LLIL stack produced different output"""
-        file_name = os.path.join(self.test_store, "jumptable_reordered")
-        self.unpackage_file(file_name)
+        file_name = self.unpackage_file("jumptable_reordered")
         bv = binja.BinaryViewType.get_view_of_file(file_name)
         reg_list = ['ch', 'cl', 'ah', 'edi', 'al', 'cx', 'ebp', 'ax', 'edx', 'ebx', 'esp', 'esi', 'dl', 'dh', 'di', 'bl', 'bh', 'eax', 'dx', 'bx', 'ecx', 'sp', 'si']
         flag_list = ['c', 'p', 'a', 'z', 's', 'o']
@@ -631,13 +627,11 @@ class TestBuilder(Builder):
                         retinfo.append("LLIL flag {} value after {}: ".format(flag, hex(ins.address)) + str(ins.get_flag_value_after(flag)))
                         retinfo.append("LLIL flag {} possible value at {}: ".format(flag, hex(ins.address)) + str(ins.get_possible_flag_values(flag)))
                         retinfo.append("LLIL flag {} possible value after {}: ".format(flag, hex(ins.address)) + str(ins.get_possible_flag_values_after(flag)))
-        os.unlink(file_name)
         return fixOutput(retinfo)
 
     def test_med_il_stack(self):
         """MLIL stack produced different output"""
-        file_name = os.path.join(self.test_store, "jumptable_reordered")
-        self.unpackage_file(file_name)
+        file_name = self.unpackage_file("jumptable_reordered")
         bv = binja.BinaryViewType.get_view_of_file(file_name)
         reg_list = ['ch', 'cl', 'ah', 'edi', 'al', 'cx', 'ebp', 'ax', 'edx', 'ebx', 'esp', 'esi', 'dl', 'dh', 'di', 'bl', 'bh', 'eax', 'dx', 'bx', 'ecx', 'sp', 'si']
         flag_list = ['c', 'p', 'a', 'z', 's', 'o']
@@ -663,13 +657,11 @@ class TestBuilder(Builder):
                         retinfo.append("MLIL flag {} value after {}: ".format(flag, hex(ins.address)) + str(ins.get_flag_value_after(flag)))
                         retinfo.append("MLIL flag {} possible value at {}: ".format(flag, hex(ins.address)) + fixSet(str(ins.get_possible_flag_values(flag))))
                         retinfo.append("MLIL flag {} possible value after {}: ".format(flag, hex(ins.address)) + fixSet(str(ins.get_possible_flag_values(flag))))
-        os.unlink(file_name)
         return fixOutput(retinfo)
 
     def test_events(self):
         """Event failure"""
-        file_name = os.path.join(self.test_store, "helloworld")
-        self.unpackage_file(file_name)
+        file_name = self.unpackage_file("helloworld")
         bv = binja.BinaryViewType['ELF'].open(file_name)
 
         results = []
@@ -679,83 +671,44 @@ class TestBuilder(Builder):
         evt = binja.AnalysisCompletionEvent(bv, simple_complete)
 
         class NotifyTest(binja.BinaryDataNotification):
-
             def data_written(self, view, offset, length):
-                def data_written_complete(self):
-                    results.append("data written: offset {0} length {1}".format(hex(offset), hex(length)))
-                evt = binja.AnalysisCompletionEvent(bv, data_written_complete)
+                results.append("data written: offset {0} length {1}".format(hex(offset), hex(length)))
 
             def data_inserted(self, view, offset, length):
-                def data_inserted_complete(self):
-                    results.append("data inserted: offset {0} length {1}".format(hex(offset), hex(length)))
-                evt = binja.AnalysisCompletionEvent(bv, data_inserted_complete)
+                results.append("data inserted: offset {0} length {1}".format(hex(offset), hex(length)))
 
             def data_removed(self, view, offset, length):
-                def data_removed_complete(self):
-                    results.append("data removed: offset {0} length {1}".format(hex(offset), hex(length)))
-                evt = binja.AnalysisCompletionEvent(bv, data_removed_complete)
+                results.append("data removed: offset {0} length {1}".format(hex(offset), hex(length)))
 
             def function_added(self, view, func):
-                def function_added_complete(self):
-                    results.append("function added: {0}".format(func.name))
-                evt = binja.AnalysisCompletionEvent(bv, function_added_complete)
+                results.append("function added: {0}".format(func.name))
 
             def function_removed(self, view, func):
-                def function_removed_complete(self):
-                    results.append("function removed: {0}".format(func.name))
-                evt = binja.AnalysisCompletionEvent(bv, function_removed_complete)
-
-            def function_updated(self, view, func):
-                def function_updated_complete(self):
-                    results.append("function updated: {0}".format(func.name))
-                evt = binja.AnalysisCompletionEvent(bv, function_updated_complete)
-
-            def function_update_requested(self, view, func):
-                def function_update_requested_complete(self):
-                    results.append("function update requested: {0}".format(func.name))
-                evt = binja.AnalysisCompletionEvent(bv, function_update_requested_complete)
+                results.append("function removed: {0}".format(func.name))
 
             def data_var_added(self, view, var):
-                def data_var_added_complete(self):
-                    results.append("data var added: {0}".format(var.name))
-                evt = binja.AnalysisCompletionEvent(bv, data_var_added_complete)
+                results.append("data var added: {0}".format(hex(var.address)))
 
             def data_var_removed(self, view, var):
-                def data_var_removed_complete(self):
-                    results.append("data var removed: {0}".format(var.name))
-                evt = binja.AnalysisCompletionEvent(bv, data_var_removed_complete)
-
-            def data_var_updated(self, view, var):
-                def data_var_updated_complete(self):
-                    results.append("data var updated: {0}".format(var.name))
-                evt = binja.AnalysisCompletionEvent(bv, data_var_updated_complete)
+                results.append("data var removed: {0}".format(hex(var.address)))
 
             def string_found(self, view, string_type, offset, length):
-                def string_found_complete(self):
-                    offset = hex(offset)
-                    length = hex(length)
-                    if offset[-1] == 'L':
-                        offset = offset[:-1]
-                    if length[-1] == 'L':
-                        length = length[:-1]
-                    results.append("string found: offset {0} length {1}".format(offset, length))
-                evt = binja.AnalysisCompletionEvent(bv, string_found_complete)
+                offset = hex(offset)
+                length = hex(length)
+                if offset[-1] == 'L':
+                    offset = offset[:-1]
+                if length[-1] == 'L':
+                    length = length[:-1]
+                results.append("string found: offset {0} length {1}".format(offset, length))
 
             def string_removed(self, view, string_type, offset, length):
-                def string_removed_complete(self):
-                    results.append("string removed: offset {0} length {1}".format(hex(offset), hex(length)))
-                evt = binja.AnalysisCompletionEvent(bv, string_removed_complete)
+                results.append("string removed: offset {0} length {1}".format(hex(offset), hex(length)))
 
             def type_defined(self, view, name, type):
-                def type_defined_complete(self):
-                    results.append("type defined: {0}".format(name))
-                evt = binja.AnalysisCompletionEvent(bv, type_defined_complete)
+                results.append("type defined: {0}".format(name))
 
             def type_undefined(self, view, name, type):
-                def type_undefined_complete(self):
-                    results.append("type undefined: {0}".format(name))
-                evt = binja.AnalysisCompletionEvent(bv, type_undefined_complete)
-
+                results.append("type undefined: {0}".format(name))
 
         test = NotifyTest()
         bv.register_notification(test)
@@ -783,21 +736,6 @@ class TestBuilder(Builder):
 
         return fixOutput(sorted(results))
 
-    def unpackage(self, fileName):
-        testname = None
-        with zipfile.ZipFile(fileName, "r") as zf:
-            testname = zf.namelist()[0]
-            zf.extractall()
-
-        if not os.path.exists(testname + ".pkl"):
-            return None, None
-        binary_oracle = pickle.load(open(testname + ".pkl", "rb"))
-        return binary_oracle.oracle_test_data, testname
-
-    def cleanup_package(self, fileName):
-        if fileName.endswith(".zip"):
-            os.unlink(fileName[:-4])
-
 
 class VerifyBuilder(Builder):
     """ The VerifyBuilder is for tests that verify
@@ -826,10 +764,9 @@ class VerifyBuilder(Builder):
         #  - Save the database
         #  - Restore the datbase
         #  - Validate that the modifications are present
-        file = os.path.join(self.test_store, "helloworld")
-        self.unpackage_file(file)
+        file_name = self.unpackage_file("helloworld")
         try:
-            bv = binja.BinaryViewType['ELF'].open(file)
+            bv = binja.BinaryViewType['ELF'].open(file_name)
             bv.update_analysis_and_wait()
             # Make some modifications to the binary view
 
@@ -844,10 +781,7 @@ class VerifyBuilder(Builder):
             bv.create_database(temp_name)
             bv.file.close()
             del bv
-        finally:
-            os.unlink(file)
 
-        try:
             bv = binja.FileMetadata(temp_name).open_existing_database(temp_name).get_view_of_type('ELF')
             bv.update_analysis_and_wait()
             bndb_functions = self.get_functions(bv)
diff --git a/suite/unit.py b/suite/unit.py
deleted file mode 100644
index e01b0f19..00000000
--- a/suite/unit.py
+++ /dev/null
@@ -1,266 +0,0 @@
-#!/usr/bin/env python
-# This is an auto generated unit test file do not edit directly
-import os
-import sys
-import unittest
-import pickle
-import zipfile
-import testcommon
-import api_test
-import difflib
-from collections import Counter
-
-global verbose
-verbose = False
-
-
-class TestBinaryNinjaAPI(unittest.TestCase):
-    # Returns a tuple of:
-    #   bool   : Two lists are equal
-    #   string : The string diff
-    # Args:
-    #   list
-    #   list   : (compare list one vs list two)
-    #   string : anything additional wanted to be printed before the string diff
-    #   bool   : the ordering of the items in the two lists must be the same
-    def report(self, oracle, test, firstText='', strictOrdering = False):
-        stringDiff = ""
-
-        equality = False
-        if not strictOrdering:
-            equality = (Counter(oracle) == Counter(test))
-        else:
-            equality = (oracle == test)
-
-        if equality:
-            return (True, '')
-        elif not strictOrdering:
-            try:
-                for elem in oracle:
-                    test.remove(elem)
-                    oracle.remove(elem)  # If it's not in the test, it won't get here!
-            except ValueError:
-                pass
-
-        differ = difflib.Differ(charjunk=difflib.IS_CHARACTER_JUNK)
-        skipped_lines = 0
-        for delta in differ.compare(oracle, test):
-            if delta[0] == ' ':
-                skipped_lines += 1
-                continue
-            if skipped_lines > 0:
-                stringDiff += "<---" + str(skipped_lines) + ' same lines--->\n'
-                skipped_lines = 0
-            delta = delta.replace('\n', '')
-            stringDiff += delta + '\n'
-
-        stringDiffList = stringDiff.split('\n')
-
-        if len(stringDiffList) > 10:
-            if not verbose:
-                stringDiff = '\n'.join(line if len(line) <= 100 else line[:100] + "...and " + str(len(line) - 100) + " more characters" for line in stringDiffList[:10])
-                stringDiff += '\n\n### And ' + str(len(stringDiffList)) + " more lines, use '-v' to show ###"
-        elif not verbose:
-            stringDiff = '\n'.join(line if len(line) <= 100 else line[:100] + "...and " + str(len(line) - 100) + " more characters" for line in stringDiffList)
-        stringDiff = '\n\n' + firstText + stringDiff
-        return (equality, stringDiff)
-
-    @classmethod
-    def setUpClass(self):
-        self.builder = testcommon.TestBuilder("suite/binaries/test_corpus")
-        try:
-            # Python 2 does not have the encodings option
-            self.oracle_test_data = pickle.load(open(os.path.join("suite", "oracle.pkl"), "rb"), encoding='charmap')
-        except TypeError:
-            self.oracle_test_data = pickle.load(open(os.path.join("suite", "oracle.pkl"), "rb"))
-        self.verifybuilder = testcommon.VerifyBuilder("suite/binaries/test_corpus")
-
-    def run_binary_test(self, testfile):
-        testname = None
-        with zipfile.ZipFile(testfile, "r") as zf:
-            testname = zf.namelist()[0]
-            zf.extractall()
-
-        self.assertTrue(os.path.exists(testname + ".pkl"), "Test pickle doesn't exist")
-        try:
-            # Python 2 does not have the encodings option
-            binary_oracle = pickle.load(open(testname + ".pkl", "rb"), encoding='charmap')
-        except TypeError:
-            binary_oracle = pickle.load(open(testname + ".pkl", "rb"))
-
-        test_builder = testcommon.BinaryViewTestBuilder(testname, "suite/binaries/test_corpus")
-        for method in test_builder.methods():
-            test = getattr(test_builder, method)()
-            oracle = binary_oracle[method]
-            if test == oracle:
-                continue
-
-            result = getattr(test_builder, method).__doc__
-            result += ":\n"
-            report = self.report(oracle, test, result)
-            self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-        os.unlink(testname)
-
-    def test_Architecture(self):
-        oracle = self.oracle_test_data['test_Architecture']
-        test = self.builder.test_Architecture()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Architecture_list(self):
-        oracle = self.oracle_test_data['test_Architecture_list']
-        test = self.builder.test_Architecture_list()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Assemble(self):
-        oracle = self.oracle_test_data['test_Assemble']
-        test = self.builder.test_Assemble()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_BinaryViewType_list(self):
-        oracle = self.oracle_test_data['test_BinaryViewType_list']
-        test = self.builder.test_BinaryViewType_list()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Enumeration(self):
-        oracle = self.oracle_test_data['test_Enumeration']
-        test = self.builder.test_Enumeration()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Function(self):
-        oracle = self.oracle_test_data['test_Function']
-        test = self.builder.test_Function()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Plugin_bin_info(self):
-        oracle = self.oracle_test_data['test_Plugin_bin_info']
-        test = self.builder.test_Plugin_bin_info()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Struct(self):
-        oracle = self.oracle_test_data['test_Struct']
-        test = self.builder.test_Struct()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_Types(self):
-        oracle = self.oracle_test_data['test_Types']
-        test = self.builder.test_Types()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_events(self):
-        oracle = self.oracle_test_data['test_events']
-        test = self.builder.test_events()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_linear_disassembly(self):
-        oracle = self.oracle_test_data['test_linear_disassembly']
-        test = self.builder.test_linear_disassembly()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_low_il_stack(self):
-        oracle = self.oracle_test_data['test_low_il_stack']
-        test = self.builder.test_low_il_stack()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_med_il_stack(self):
-        oracle = self.oracle_test_data['test_med_il_stack']
-        test = self.builder.test_med_il_stack()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_partial_register_dataflow(self):
-        oracle = self.oracle_test_data['test_partial_register_dataflow']
-        test = self.builder.test_partial_register_dataflow()
-        report = self.report(oracle, test)
-        self.assertTrue(report[0], report[1])  # Test does not agree with oracle
-
-    def test_verify_BNDB_round_trip(self):
-        self.assertTrue(self.verifybuilder.test_verify_BNDB_round_trip(), self.test_verify_BNDB_round_trip.__doc__)
-
-    def test_binary___aliased_jumptable(self):
-        self.run_binary_test('suite/binaries/test_corpus/aliased_jumptable.zip')
-
-    def test_binary___byte_jump_table(self):
-        self.run_binary_test('suite/binaries/test_corpus/byte_jump_table.zip')
-
-    def test_binary___duff(self):
-        self.run_binary_test('suite/binaries/test_corpus/duff.zip')
-
-    def test_binary___helloworld(self):
-        self.run_binary_test('suite/binaries/test_corpus/helloworld.zip')
-
-    def test_binary___helloworld_armeb(self):
-        self.run_binary_test('suite/binaries/test_corpus/helloworld_armeb.zip')
-
-    def test_binary___integer_test(self):
-        self.run_binary_test('suite/binaries/test_corpus/integer_test.zip')
-
-    def test_binary___interprocedural_alias(self):
-        self.run_binary_test('suite/binaries/test_corpus/interprocedural_alias.zip')
-
-    def test_binary___jump_loop(self):
-        self.run_binary_test('suite/binaries/test_corpus/jump_loop.zip')
-
-    def test_binary___jumptable_aarch64(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_aarch64.zip')
-
-    def test_binary___jumptable_mips32(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_mips32.zip')
-
-    def test_binary___jumptable_multiple_indirect(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_multiple_indirect.zip')
-
-    def test_binary___jumptable_no_range_check(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_no_range_check.zip')
-
-    def test_binary___jumptable_reordered(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_reordered.zip')
-
-    def test_binary___jumptable_x86(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_x86.zip')
-
-    def test_binary___jumptable_x86_64(self):
-        self.run_binary_test('suite/binaries/test_corpus/jumptable_x86_64.zip')
-
-    def test_binary___loop_constant_propagate(self):
-        self.run_binary_test('suite/binaries/test_corpus/loop_constant_propagate.zip')
-
-    def test_binary___partial_register_dataflow(self):
-        self.run_binary_test('suite/binaries/test_corpus/partial_register_dataflow.zip')
-
-    def test_binary___pe_thumb(self):
-        self.run_binary_test('suite/binaries/test_corpus/pe_thumb.zip')
-
-    def test_binary___quick3dcoreplugin_dll(self):
-        self.run_binary_test('suite/binaries/test_corpus/quick3dcoreplugin.dll.zip')
-
-    def test_binary___rangecheck(self):
-        self.run_binary_test('suite/binaries/test_corpus/rangecheck.zip')
-
-    def test_binary___switch_linux_ppc_le_32(self):
-        self.run_binary_test('suite/binaries/test_corpus/switch_linux_ppc_le_32.zip')
-
-    def test_binary___x87(self):
-        self.run_binary_test('suite/binaries/test_corpus/x87.zip')
-
-
-if __name__ == "__main__":
-    if len(sys.argv) > 1:
-        if sys.argv[1] == '-v' or sys.argv[1] == '-V' or sys.argv[1] == '--verbose':
-            verbose = True
-
-    test_suite = unittest.defaultTestLoader.loadTestsFromModule(api_test)
-    test_suite.addTest(unittest.defaultTestLoader.loadTestsFromTestCase(TestBinaryNinjaAPI))
-    runner = unittest.TextTestRunner(verbosity=2)
-    runner.run(test_suite)