Large PDB parsing performance improvements

5 files changed, 88 insertions, 76 deletions

diff --git a/rust/Cargo.lock b/rust/Cargo.lock
index e45d2da2..dfccee5a 100644
--- a/rust/Cargo.lock
+++ b/rust/Cargo.lock
@@ -458,9 +458,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.2.5"
+version = "2.2.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4"
+checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
 dependencies = [
  "equivalent",
  "hashbrown",
diff --git a/rust/examples/pdb-ng/src/lib.rs b/rust/examples/pdb-ng/src/lib.rs
index 95407636..e2e3ec44 100644
--- a/rust/examples/pdb-ng/src/lib.rs
+++ b/rust/examples/pdb-ng/src/lib.rs
@@ -364,13 +364,15 @@ impl PDBParser {
     ) -> Result<()> {
         let mut pdb = PDB::open(Cursor::new(&conts))?;
 
+        let settings = Settings::new("");
+
         if let Some(info) = parse_pdb_info(view) {
-            let pdb_info = pdb.pdb_information()?;
+            let pdb_info = &pdb.pdb_information()?;
             if info.guid.as_slice() != pdb_info.guid.as_ref() {
                 if check_guid {
                     return Err(anyhow!("PDB GUID does not match"));
                 } else {
-                    let ask = Settings::new("").get_string(
+                    let ask = settings.get_string(
                         "pdb.features.loadMismatchedPDB",
                         Some(view),
                         None,
@@ -406,7 +408,7 @@ impl PDBParser {
                 }
             }
 
-            if did_download && Settings::new("").get_bool("pdb.files.localStoreCache", None, None) {
+            if did_download && settings.get_bool("pdb.files.localStoreCache", None, None) {
                 match active_local_cache(Some(view)) {
                     Ok(cache) => {
                         let mut cab_path = PathBuf::from(&cache);
@@ -485,7 +487,7 @@ impl PDBParser {
             if check_guid {
                 return Err(anyhow!("File not compiled with PDB information"));
             } else {
-                let ask = Settings::new("").get_string(
+                let ask = settings.get_string(
                     "pdb.features.loadMismatchedPDB",
                     Some(view),
                     None,
diff --git a/rust/examples/pdb-ng/src/parser.rs b/rust/examples/pdb-ng/src/parser.rs
index 9e0bf025..11bad268 100644
--- a/rust/examples/pdb-ng/src/parser.rs
+++ b/rust/examples/pdb-ng/src/parser.rs
@@ -12,9 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use std::collections::{BTreeMap, BTreeSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 use std::env;
 use std::fmt::Display;
+use std::sync::OnceLock;
 
 use anyhow::{anyhow, Result};
 use log::{debug, info};
@@ -200,7 +201,8 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
                 self.settings
                     .get_bool("pdb.features.allowVoidGlobals", Some(self.bv), None);
 
-            for sym in symbols {
+            let min_confidence_type = Conf::new(Type::void(), min_confidence());
+            for sym in symbols.iter() {
                 match sym {
                     ParsedSymbol::Data(ParsedDataSymbol {
                         address,
@@ -209,7 +211,7 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
                         ..
                     }) => {
                         let real_type =
-                            type_.unwrap_or_else(|| Conf::new(Type::void(), min_confidence()));
+                            type_.as_ref().unwrap_or(&min_confidence_type);
 
                         if real_type.contents.type_class() == TypeClass::VoidTypeClass {
                             if !allow_void {
@@ -228,10 +230,10 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
                         });
                         self.debug_info
                             .add_data_variable_info(DataVariableAndName::new(
-                                address,
-                                real_type,
+                                *address,
+                                real_type.clone(),
                                 true,
-                                name.full_name.unwrap_or(name.raw_name),
+                                name.full_name.as_ref().unwrap_or(&name.raw_name),
                             ));
                     }
                     s => {
@@ -282,19 +284,11 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
     fn collect_name(
         &self,
         name: &NamedTypeReference,
-        unknown_names: &mut BTreeMap<String, NamedTypeReferenceClass>,
+        unknown_names: &mut HashMap<String, NamedTypeReferenceClass>,
     ) {
         let used_name = name.name().to_string();
         if let Some(&found) =
-            unknown_names.iter().find_map(
-                |(key, value)| {
-                    if key == &used_name {
-                        Some(value)
-                    } else {
-                        None
-                    }
-                },
-            )
+            unknown_names.get(&used_name)
         {
             if found != name.class() {
                 // Interesting case, not sure we care
@@ -315,7 +309,7 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
     fn collect_names(
         &self,
         ty: &Type,
-        unknown_names: &mut BTreeMap<String, NamedTypeReferenceClass>,
+        unknown_names: &mut HashMap<String, NamedTypeReferenceClass>,
     ) {
         match ty.type_class() {
             TypeClass::StructureTypeClass => {
@@ -366,13 +360,13 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
         symbols: &Vec<ParsedSymbol>,
         progress: Box<dyn Fn(usize, usize) -> Result<()> + '_>,
     ) -> Result<()> {
-        let mut unknown_names = BTreeMap::new();
+        let mut unknown_names = HashMap::new();
         let mut known_names = self
             .bv
             .types()
             .iter()
             .map(|qnat| qnat.name().string())
-            .collect::<BTreeSet<_>>();
+            .collect::<HashSet<_>>();
 
         for ty in &self.named_types {
             known_names.insert(ty.0.clone());
@@ -404,7 +398,7 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
         }
 
         for (name, class) in unknown_names.into_iter() {
-            if known_names.iter().any(|known| known == &name) {
+            if known_names.contains(&name) {
                 self.log(|| format!("Found referenced name and ignoring: {}", &name));
                 continue;
             }
@@ -460,7 +454,11 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
 
     /// Lazy logging function that prints like 20MB of messages
     pub(crate) fn log<F: FnOnce() -> D, D: Display>(&self, msg: F) {
-        if env::var("BN_DEBUG_PDB").is_ok() {
+        static MEM: OnceLock<bool> = OnceLock::new();
+        let debug_pdb = MEM.get_or_init(|| {
+            env::var("BN_DEBUG_PDB").is_ok()
+        });
+        if *debug_pdb {
             let space = "\t".repeat(self.type_stack.len()) + &"\t".repeat(self.symbol_stack.len());
             let msg = format!("{}", msg());
             debug!(
diff --git a/rust/examples/pdb-ng/src/symbol_parser.rs b/rust/examples/pdb-ng/src/symbol_parser.rs
index 5a1d8761..a80f4841 100644
--- a/rust/examples/pdb-ng/src/symbol_parser.rs
+++ b/rust/examples/pdb-ng/src/symbol_parser.rs
@@ -14,6 +14,7 @@
 
 use std::collections::{BTreeMap, HashMap, HashSet};
 use std::mem;
+use std::sync::OnceLock;
 
 use anyhow::{anyhow, Result};
 use itertools::Itertools;
@@ -515,7 +516,7 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
 
                 self.log(|| format!("Symbol {:?} ", sym.index()));
                 let (name, address) =
-                    if let Some(parsed) = self.handle_symbol_index(sym.index(), sym)? {
+                    if let Some(parsed) = self.handle_symbol_index(sym.index(), &sym)? {
                         final_symbols.insert(sym.index());
                         match parsed {
                             ParsedSymbol::Data(ParsedDataSymbol { name, address, .. }) => {
@@ -617,17 +618,14 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
     fn handle_symbol_index(
         &mut self,
         idx: SymbolIndex,
-        sym: Symbol,
+        sym: &Symbol,
     ) -> Result<Option<&ParsedSymbol>> {
         if let None = self.indexed_symbols.get(&idx) {
             match sym.parse() {
                 Ok(data) => match self.handle_symbol(idx, &data) {
                     Ok(Some(parsed)) => {
                         self.log(|| format!("Symbol {} parsed into: {:?}", idx, parsed));
-                        match &parsed {
-                            _ => {}
-                        }
-                        self.indexed_symbols.insert(idx, parsed.clone());
+                        self.indexed_symbols.insert(idx, parsed);
                     }
                     Ok(None) => {}
                     e => {
@@ -1786,7 +1784,7 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
         raw_name: &String,
         rva: Rva,
     ) -> Result<(Option<Conf<Ref<Type>>>, Option<QualifiedName>)> {
-        let (mut t, mut name) = match demangle_ms(&self.arch, raw_name.clone(), true) {
+        let (mut t, mut name) = match demangle_ms(&self.arch, raw_name, true) {
             Ok((Some(t), name)) => (Some(Conf::new(t, DEMANGLE_CONFIDENCE)), name),
             Ok((_, name)) => (None, name),
             _ => (None, vec![raw_name.clone()]),
@@ -1818,48 +1816,51 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
 
         // These have types but they aren't actually set anywhere. So it's the demangler's
         // job to take care of them, apparently?
-        let name_to_type: HashMap<String, Vec<String>> = HashMap::from_iter([
-            (
-                "`RTTI Complete Object Locator'".to_string(),
-                vec![
-                    "_s_RTTICompleteObjectLocator".to_string(),
-                    "_s__RTTICompleteObjectLocator".to_string(),
-                    "_s__RTTICompleteObjectLocator2".to_string(),
-                ],
-            ),
-            (
-                "`RTTI Class Hierarchy Descriptor'".to_string(),
-                vec![
-                    "_s_RTTIClassHierarchyDescriptor".to_string(),
-                    "_s__RTTIClassHierarchyDescriptor".to_string(),
-                    "_s__RTTIClassHierarchyDescriptor2".to_string(),
-                ],
-            ),
-            (
-                // TODO: This type is dynamic
-                "`RTTI Base Class Array'".to_string(),
-                vec![
-                    "_s_RTTIBaseClassArray".to_string(),
-                    "_s__RTTIBaseClassArray".to_string(),
-                    "_s__RTTIBaseClassArray2".to_string(),
-                ],
-            ),
-            (
-                "`RTTI Base Class Descriptor at (".to_string(),
-                vec![
-                    "_s_RTTIBaseClassDescriptor".to_string(),
-                    "_s__RTTIBaseClassDescriptor".to_string(),
-                    "_s__RTTICBaseClassDescriptor2".to_string(),
-                ],
-            ),
-            (
-                "`RTTI Type Descriptor'".to_string(),
-                vec!["_TypeDescriptor".to_string()],
-            ),
-        ]);
+        static MEM: OnceLock<Vec<(String, Vec<String>)>> = OnceLock::new();
+        let name_to_type = MEM.get_or_init(|| {
+            vec![
+                (
+                    "`RTTI Complete Object Locator'".to_string(),
+                    vec![
+                        "_s_RTTICompleteObjectLocator".to_string(),
+                        "_s__RTTICompleteObjectLocator".to_string(),
+                        "_s__RTTICompleteObjectLocator2".to_string(),
+                    ],
+                ),
+                (
+                    "`RTTI Class Hierarchy Descriptor'".to_string(),
+                    vec![
+                        "_s_RTTIClassHierarchyDescriptor".to_string(),
+                        "_s__RTTIClassHierarchyDescriptor".to_string(),
+                        "_s__RTTIClassHierarchyDescriptor2".to_string(),
+                    ],
+                ),
+                (
+                    // TODO: This type is dynamic
+                    "`RTTI Base Class Array'".to_string(),
+                    vec![
+                        "_s_RTTIBaseClassArray".to_string(),
+                        "_s__RTTIBaseClassArray".to_string(),
+                        "_s__RTTIBaseClassArray2".to_string(),
+                    ],
+                ),
+                (
+                    "`RTTI Base Class Descriptor at (".to_string(),
+                    vec![
+                        "_s_RTTIBaseClassDescriptor".to_string(),
+                        "_s__RTTIBaseClassDescriptor".to_string(),
+                        "_s__RTTICBaseClassDescriptor2".to_string(),
+                    ],
+                ),
+                (
+                    "`RTTI Type Descriptor'".to_string(),
+                    vec!["_TypeDescriptor".to_string()],
+                ),
+            ]
+        });
 
         if let Some(last_name) = name.last() {
-            for (search_name, search_types) in &name_to_type {
+            for (search_name, search_types) in name_to_type.iter() {
                 if last_name.contains(search_name) {
                     for search_type in search_types {
                         if let Some(ty) = self.named_types.get(search_type) {
diff --git a/rust/examples/pdb-ng/src/type_parser.rs b/rust/examples/pdb-ng/src/type_parser.rs
index f21a04fa..caf732b2 100644
--- a/rust/examples/pdb-ng/src/type_parser.rs
+++ b/rust/examples/pdb-ng/src/type_parser.rs
@@ -13,6 +13,7 @@
 // limitations under the License.
 
 use std::collections::HashMap;
+use std::sync::OnceLock;
 
 use anyhow::{anyhow, Result};
 use binaryninja::architecture::{Architecture, CoreArchitecture};
@@ -410,12 +411,22 @@ impl<'a, S: Source<'a> + 'a> PDBParserInstance<'a, S> {
                 self.log(|| format!("Remove builtin type {}", name));
             }
         }
+
+        static MEM: OnceLock<Regex> = OnceLock::new();
+        let uint_regex = MEM.get_or_init(|| {
+            Regex::new(r"u?int\d+_t").unwrap()
+        });
+
+        let float_regex = MEM.get_or_init(|| {
+            Regex::new(r"float\d+").unwrap()
+        });
+
         let mut remove_names = vec![];
         for (name, _) in &self.named_types {
-            if Regex::new(r"u?int\d+_t")?.is_match(name) {
+            if uint_regex.is_match(name) {
                 remove_names.push(name.clone());
             }
-            if Regex::new(r"float\d+")?.is_match(name) {
+            if float_regex.is_match(name) {
                 remove_names.push(name.clone());
             }
         }