feat(perf): better search performance by avoiding allocations (#40)

* perf(search): Initial optimization round which improves searches

* feat(test): Switched to grcov for code coverage

Author: PyDataBlog

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Ask a question
-    url: https://github.com/bebr/simstring_rs/discussions/new
+    url: https://github.com/PyDataBlog/simstring_rs/discussions/new
     about: Please ask and answer questions here.

.github/PULL_REQUEST_TEMPLATE.md

@@ -4,6 +4,7 @@ about: Propose a change to the project
 title: "feat(scope): describe your change"
 labels: ""
 assignees: ""
+
 ---
 
 **Description**

.github/workflows/coverage.yml

@@ -31,18 +31,27 @@ jobs:
         with:
           python-version: "3.13"
 
-      - name: Install cargo-tarpaulin
-        run: cargo install cargo-tarpaulin --version 0.26.0 --force
-
-      - name: Set Python interpreter for pyo3
-        run: echo "PYTHON_SYS_EXECUTABLE=$(which python3)" >> $GITHUB_ENV
+      - name: Install grcov and llvm-tools
+        run: |
+          cargo install grcov --force
+          rustup component add llvm-tools
 
       - name: Run tests and generate coverage report
+        env:
+          SIMSTRING_RS_COVERAGE: "1"
         run: |
-          LIBDIR=$(python3 -c "import sysconfig; print(sysconfig.get_config_var('LIBDIR'))")
-          VERSION=$(python3 -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
-          export RUSTFLAGS="-L $LIBDIR -lpython$VERSION"
-          cargo tarpaulin --all-features --out Xml --output-dir ./coverage
+          # Create directory for coverage reports
+          mkdir -p ./coverage
+          # Set flags for coverage generation
+          export CARGO_INCREMENTAL=0
+          export RUSTFLAGS="-Cinstrument-coverage"
+          # Set the path for the raw coverage data
+          export LLVM_PROFILE_FILE="target/coverage/simstring_rs-%p-%m.profraw"
+          # Run all tests, including the ignored python bindings
+          cargo test --all-features -- --include-ignored
+          # Generate the coverage report
+          grcov . --binary-path ./target/debug/ -s . -t lcov --branch --ignore-not-existing --ignore "tests/*" --ignore "examples/*" --ignore "build.rs" -o ./coverage.lcov
+          grcov . --binary-path ./target/debug/ -s . -t cobertura --branch --ignore-not-existing --ignore "tests/*" --ignore "examples/*" --ignore "build.rs" -o ./coverage/cobertura.xml
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v5

Cargo.lock

@@ -18,10 +18,11 @@ path = "src/lib.rs"
 crate-type = ["cdylib", "rlib"]
 
 [dependencies]
-rayon = "1.10"
 ahash = "0.8"
+rayon = "1.10"
+thiserror = "2.0"
 lasso = "0.7"
-thiserror = "2"
+rustc-hash = "2.1"
 pyo3 = { version = "0.25", features = ["extension-module", "abi3-py37"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"

Cargo.toml

@@ -2,6 +2,8 @@
 
 [![Build Status](https://github.com/PyDataBlog/simstring_rs/actions/workflows/CI.yml/badge.svg)](https://github.com/PyDataBlog/simstring_rs/actions)
 [![Crates.io](https://img.shields.io/crates/v/simstring_rust.svg)](https://crates.io/crates/simstring_rust)
+[![PyPI version](https://badge.fury.io/py/simstring-rust.svg)](https://badge.fury.io/py/simstring-rust)
+[![Python versions](https://img.shields.io/pypi/pyversions/simstring-rust.svg)](https://pypi.org/project/simstring-rust)
 [![Documentation](https://docs.rs/simstring_rust/badge.svg)](https://docs.rs/simstring_rust)
 [![Rust](https://img.shields.io/badge/rust-1.63.0%2B-blue.svg?maxAge=3600)](https://github.com/PyDataBlog/simstring_rs)
 [![Codecov](https://img.shields.io/codecov/c/github/PyDataBlog/simstring_rs?token=XJM8O8TD4U)](https://codecov.io/gh/PyDataBlog/simstring_rs)

README.md

@@ -20,6 +20,7 @@ classifiers = [
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "License :: OSI Approved :: MIT License",
     "Operating System :: POSIX",
     "Operating System :: MacOS :: MacOS X",

pyproject.toml

@@ -1,15 +1,15 @@
 use crate::database::{Database, StringId};
 use crate::extractors::FeatureExtractor;
-use ahash::{AHashMap, AHashSet};
 use lasso::{Rodeo, Spur};
+use rustc_hash::{FxHashMap, FxHashSet};
 use std::fmt;
 use std::sync::{Arc, Mutex};
 
 pub struct HashDb {
     feature_extractor: Arc<dyn FeatureExtractor>,
     pub strings: Vec<String>,
     string_features: Vec<Vec<Spur>>,
-    feature_map: AHashMap<usize, AHashMap<Spur, AHashSet<StringId>>>,
+    feature_map: FxHashMap<usize, FxHashMap<Spur, FxHashSet<StringId>>>,
     interner: Arc<Mutex<Rodeo>>,
 }
 
@@ -37,7 +37,7 @@ impl HashDb {
             feature_extractor,
             strings: Vec::new(),
             string_features: Vec::new(),
-            feature_map: AHashMap::default(),
+            feature_map: FxHashMap::default(),
             interner: Arc::new(Mutex::new(Rodeo::default())),
         }
     }
@@ -75,7 +75,7 @@ impl Database for HashDb {
         self.interner.lock().unwrap().clear();
     }
 
-    fn lookup_strings(&self, size: usize, feature: Spur) -> Option<&AHashSet<StringId>> {
+    fn lookup_strings(&self, size: usize, feature: Spur) -> Option<&FxHashSet<StringId>> {
         self.feature_map.get(&size)?.get(&feature)
     }
 
@@ -98,4 +98,8 @@ impl Database for HashDb {
     fn interner(&self) -> Arc<Mutex<Rodeo>> {
         Arc::clone(&self.interner)
     }
+
+    fn total_strings(&self) -> usize {
+        self.strings.len()
+    }
 }

src/database/hashdb.rs

@@ -1,21 +1,22 @@
 mod hashdb;
 
 use crate::extractors::FeatureExtractor;
-use ahash::AHashSet;
 use lasso::{Rodeo, Spur};
+use rustc_hash::FxHashSet;
 use std::sync::{Arc, Mutex};
 
 pub type StringId = usize;
 
 pub trait Database: Send + Sync {
     fn insert(&mut self, text: String);
     fn clear(&mut self);
-    fn lookup_strings(&self, size: usize, feature: Spur) -> Option<&AHashSet<StringId>>;
+    fn lookup_strings(&self, size: usize, feature: Spur) -> Option<&FxHashSet<StringId>>;
     fn get_string(&self, id: StringId) -> Option<&str>;
     fn get_features(&self, id: StringId) -> Option<&Vec<Spur>>;
     fn feature_extractor(&self) -> &dyn FeatureExtractor;
     fn max_feature_len(&self) -> usize;
     fn interner(&self) -> Arc<Mutex<Rodeo>>;
+    fn total_strings(&self) -> usize;
 }
 
 pub use hashdb::HashDb;

src/database/mod.rs

@@ -27,15 +27,36 @@ impl FeatureExtractor for CharacterNgrams {
         if self.n == 0 {
             return vec![];
         }
-        let padding = self.endmarker.repeat(self.n.saturating_sub(1));
-        let padded_text = format!("{padding}{text}{padding}");
-
-        let ngrams: Vec<String> = padded_text
-            .chars()
-            .collect::<Vec<char>>()
-            .windows(self.n)
-            .map(|window| window.iter().collect())
-            .collect();
+
+        // Pre-calculate capacity to avoid reallocations
+        let text_len = text.chars().count();
+        let padding_len = self.n.saturating_sub(1);
+        let total_len = text_len + 2 * padding_len;
+
+        if total_len < self.n {
+            return vec![];
+        }
+
+        let expected_ngrams = total_len - self.n + 1;
+        let mut ngrams = Vec::with_capacity(expected_ngrams);
+
+        let padding = self.endmarker.repeat(padding_len);
+
+        // collect chars once, then slice
+        let mut all_chars = Vec::with_capacity(total_len);
+        all_chars.extend(padding.chars());
+        all_chars.extend(text.chars());
+        all_chars.extend(padding.chars());
+
+        // Generate n-grams using efficient windowing
+        for window in all_chars.windows(self.n) {
+            // Pre-allocate string with known capacity
+            let mut ngram = String::with_capacity(self.n * 4); // Assume max 4 bytes per char
+            for &ch in window {
+                ngram.push(ch);
+            }
+            ngrams.push(ngram);
+        }
 
         super::append_feature_counts(interner, ngrams)
     }