Several documentation improvements and fixes

- Reduce file count, add minifier script

- Group UI and group Core C

- Add more items to groups

- Add Tranform docs

3 files changed, 187 insertions, 13 deletions

diff --git a/api-docs/cppdocs/Doxyfile b/api-docs/cppdocs/Doxyfile
index 244c3163..9c33604c 100644
--- a/api-docs/cppdocs/Doxyfile
+++ b/api-docs/cppdocs/Doxyfile
@@ -384,7 +384,7 @@ IDL_PROPERTY_SUPPORT   = YES
 # all members of a group must be documented explicitly.
 # The default value is: NO.
 
-DISTRIBUTE_GROUP_DOC   = YES
+DISTRIBUTE_GROUP_DOC   = NO
 
 # If one adds a struct or class to a group and this option is enabled, then also
 # any nested class or struct is added to the same group. By default this option
@@ -640,7 +640,7 @@ SORT_MEMBERS_CTORS_1ST = NO
 # appear in their defined order.
 # The default value is: NO.
 
-SORT_GROUP_NAMES       = NO
+SORT_GROUP_NAMES       = YES
 
 # If the SORT_BY_SCOPE_NAME tag is set to YES, the class list will be sorted by
 # fully-qualified names, including namespaces. If set to NO, the class list will
@@ -709,14 +709,14 @@ MAX_INITIALIZER_LINES  = 30
 # list will mention the files that were used to generate the documentation.
 # The default value is: YES.
 
-SHOW_USED_FILES        = YES
+SHOW_USED_FILES        = NO
 
 # Set the SHOW_FILES tag to NO to disable the generation of the Files page. This
 # will remove the Files entry from the Quick Index and from the Folder Tree View
 # (if specified).
 # The default value is: YES.
 
-SHOW_FILES             = YES
+SHOW_FILES             = NO
 
 # Set the SHOW_NAMESPACES tag to NO to disable the generation of the Namespaces
 # page. This will remove the Namespaces entry from the Quick Index and from the
@@ -904,7 +904,17 @@ EXCLUDE_PATTERNS       =
 # Note that the wildcards are matched against the file with absolute path, so to
 # exclude all test directories use the pattern */test/*
 
-EXCLUDE_SYMBOLS        =
+# These symbols by themselves generate 60MB and 1500 pages of documentation.
+# For shipping with builds this is not reasonable.
+
+EXCLUDE_SYMBOLS        = BinaryNinja::LowLevelILInstructionAccessor \
+                         BinaryNinja::LowLevelILInstructionAccessor* \
+                         BinaryNinja::MediumLevelILInstructionAccessor \
+                         BinaryNinja::MediumLevelILInstructionAccessor* \
+                         BinaryNinja::HighLevelILInstructionAccessor \
+                         BinaryNinja::HighLevelILInstructionAccessor* \
+
+
 #EXCLUDE_SYMBOLS        = QProgressIndicator #Broke Exhale
 
 # The EXAMPLE_PATH tag can be used to specify one or more files or directories
@@ -1084,7 +1094,7 @@ VERBATIM_HEADERS       = NO
 # classes, structs, unions or interfaces.
 # The default value is: YES.
 
-ALPHABETICAL_INDEX     = YES
+ALPHABETICAL_INDEX     = NO
 
 # The COLS_IN_ALPHA_INDEX tag can be used to specify the number of columns in
 # which the alphabetical index list will be split.
@@ -2225,7 +2235,7 @@ EXTERNAL_PAGES         = YES
 # powerful graphs.
 # The default value is: YES.
 
-CLASS_DIAGRAMS         = YES
+CLASS_DIAGRAMS         = NO
 
 # You can include diagrams made with dia in doxygen documentation. Doxygen will
 # then run dia to produce the diagram and insert it in the documentation. The
@@ -2298,14 +2308,14 @@ CLASS_GRAPH            = NO
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS tag is set to YES then doxygen will generate a graph for
 # groups, showing the direct groups dependencies.
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-GROUP_GRAPHS           = YES
+GROUP_GRAPHS           = NO
 
 # If the UML_LOOK tag is set to YES, doxygen will generate inheritance and
 # collaboration diagrams in a style similar to the OMG's Unified Modeling
@@ -2383,7 +2393,7 @@ CALLER_GRAPH           = NO
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-GRAPHICAL_HIERARCHY    = YES
+GRAPHICAL_HIERARCHY    = NO
 
 # If the DIRECTORY_GRAPH tag is set to YES then doxygen will show the
 # dependencies a directory has on other directories in a graphical way. The
@@ -2392,7 +2402,7 @@ GRAPHICAL_HIERARCHY    = YES
 # The default value is: YES.
 # This tag requires that the tag HAVE_DOT is set to YES.
 
-DIRECTORY_GRAPH        = YES
+DIRECTORY_GRAPH        = NO
 
 # The DOT_IMAGE_FORMAT tag can be used to set the image format of the images
 # generated by dot. For an explanation of the image formats see the section
diff --git a/api-docs/cppdocs/README.md b/api-docs/cppdocs/README.md
index 56412839..615518ee 100644
--- a/api-docs/cppdocs/README.md
+++ b/api-docs/cppdocs/README.md
@@ -19,6 +19,8 @@ The theme for our documentation works best with doxygen 1.9.0 - 1.9.4
 ### macOS (Homebrew)
 
 ```
-wget https://raw.githubusercontent.com/Homebrew/homebrew-core/41828ee36b96e35b63b2a4c8cfc2df2c3728944a/Formula/doxygen.rb`
-brew install ./doxygen.rb
+$ wget https://raw.githubusercontent.com/Homebrew/homebrew-core/41828ee36b96e35b63b2a4c8cfc2df2c3728944a/Formula/doxygen.rb
+$ shasum -a 256 doxygen.rb
+4d1294c815cf0f76c55b14c5f47c25f523bd860a7cc9b077cce9589d84678396  doxygen.rb
+$ brew install ./doxygen.rb
 ```
diff --git a/api-docs/cppdocs/build_min_docs.py b/api-docs/cppdocs/build_min_docs.py
new file mode 100644
index 00000000..455c1376
--- /dev/null
+++ b/api-docs/cppdocs/build_min_docs.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+
+# =-=--
+# This script builds the documentation with doxygen and runs minification routines.
+# This allows shipping a few thousand fewer files in updates including C++ documentation.
+#
+# Usage:
+# install doxygen 1.9.4 (or this script will not work. running minification on a different version can cause
+# 						 real issues.)
+# python3 build_min_docs.py
+# =-=--
+
+__DOXYGEN_REQUIRED_VERSION__ = "1.9.4"
+
+import os
+import sys
+import json
+from collections import namedtuple
+from typing import List
+import subprocess
+import shutil
+
+
+def system_with_output(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE):
+	proc = subprocess.Popen("" + cmd,
+							stdout=stdout,
+							stderr=stderr,
+							shell=True,
+							universal_newlines=True)
+	std_out, std_err = proc.communicate()
+	return proc.returncode, std_out, std_err
+
+
+deletion_queue = []
+
+
+NavItemEntry = namedtuple("NavItemEntry", ["dName", "htmlLink", "vName"])
+
+
+# Load the singular variable in a file as a list of navitems
+def pop_var(filename):
+	"""
+	Load in the file data and entries within a file.
+
+	File is in the format `var varName = <json valid array>`
+	if <json_valid_array>[n][2] is a list, item[n][2] is a subfile, and we want to
+		load the entry and return it.
+
+	:param filename: Filename to load the entries and data from.
+	:return: Tuple containing a list of NavItemEntrys and the data in the file.
+	"""
+	fp = open(filename)
+	file_data = fp.read()
+	fp.close()
+	data_string = file_data.split(' =', 1)[1][:-1]
+	data = json.loads(data_string)
+	items = []
+	for entry in data:
+		if not isinstance(entry[2], list):
+			items.append(NavItemEntry(entry[0], entry[1], entry[2]))
+	return items, file_data
+
+
+def load_items_in_file(filename):
+	"""
+	Recursively load in the text of file {filename} and all subfiles referenced by it.
+
+	:param filename: Target root filename. e.g. 'modules.js'
+	:return: Combined text of javascript file tree.
+	"""
+	items = []
+	sub_items, file_data = pop_var(filename)
+	items.append(file_data)
+	deletion_queue.append(filename)
+	for item in sub_items:
+		if item.vName is not None:
+			items += load_items_in_file("html/" + item.vName + ".js")
+	return items
+
+
+def minifier():
+
+	# Typically, doxygen's navbar will lazy load the data in all of these variables.
+	# While this has miniscule performance benefits, it generates thousands of js files.
+	# Here, we take all js variables that the navbar will ever be able to load, and
+	# 	add them to the top of the navbar.js file itself.
+
+	navtree_built_data = ""
+	for mod in load_items_in_file("html/modules.js"):
+		navtree_built_data += mod + "\n"
+	for mod in load_items_in_file("html/namespaces.js"):
+		navtree_built_data += mod + "\n"
+	for mod in load_items_in_file("html/annotated.js"):
+		navtree_built_data += mod + "\n"
+
+	# The navtree indices also need to be loaded in since we're modifying how navbar.js::getScript works.
+	# This also saves another ~60 files.
+	for nav_tree_index_file in os.listdir("html"):
+		if 'navtreeindex' in nav_tree_index_file:
+			with open("html/" + nav_tree_index_file, "r") as fp:
+				navtree_built_data += fp.read() + "\n"
+				deletion_queue.append("html/" + nav_tree_index_file)
+
+	while "\n\n" in navtree_built_data:
+		navtree_built_data = navtree_built_data.replace("\n\n", "\n")
+
+	navtree_built_data = navtree_built_data.replace("\n", "")
+
+	fp = open("html/navtree.js", "r")
+	navtree_orig = fp.read()
+	fp.close()
+
+	# getScript(scriptName,func,show) here originally loads the js file and calls func once that is complete
+	# Here, we just want to skip the whole process and immediately call the callback.
+	nav_tree_fixed_get_script = "function getScript(scriptName,func,show) { func(); }"
+
+	navtree_before_get_script = navtree_orig.split("function getScript(scriptName,func,show)")[0]
+	navtree_after_get_script = navtree_orig.split("function getScript(scriptName,func,show)")[1].split('}', 1)[1]
+
+	nav_tree_fixed = navtree_before_get_script + nav_tree_fixed_get_script + navtree_after_get_script
+	navtree = navtree_built_data + "\n" + nav_tree_fixed
+
+	fp = open("html/navtree.js", "w")
+	fp.write(navtree)
+	fp.close()
+
+
+def build_doxygen():
+	if not os.path.exists('./Doxyfile'):
+		print('No Doxyfile found. Are you in the right directory?')
+		sys.exit(1)
+	_, vers, _ = system_with_output("doxygen -V")
+	if __DOXYGEN_REQUIRED_VERSION__ not in vers.strip():
+		print(f'Please use Doxygen {__DOXYGEN_REQUIRED_VERSION__} to build documentation')
+		sys.exit(1)
+
+	if os.path.exists('./html/'):
+		print('Clearing ./html/')
+		try:
+			shutil.rmtree("./html/")
+		except OSError:
+			# doing it twice works (on macOS) ¯\_(ツ)_/¯
+			shutil.rmtree("./html/")
+	print(f'Building doxygen docs...')
+	stat, out, err = system_with_output("doxygen Doxyfile")
+	print(f"Built Doxygen with status code {stat}")
+	print("Output dir is ./html/")
+
+
+def main():
+	build_doxygen()
+	print("Minifying Output")
+	minifier()
+	for file in deletion_queue:
+		file = "./" + file
+		os.remove(file)
+	print(f'Was able to clear {len(deletion_queue)} "redundant" files')
+	print(f'Done. Output is in ./html/')
+
+
+if __name__ == "__main__":
+	main()