Case-Insensitive UTF-8 Search with AVX-512 🌾🪡🌾

.github/workflows/prerelease.yml

@@ -110,6 +110,7 @@ jobs:
         uses: actions/setup-go@v6
         with:
           go-version: ">=1.24"
+          cache: false
       - name: Test Go binding
         working-directory: golang
         env:
@@ -169,11 +170,8 @@ jobs:
         run: npm install && npm test
 
       # Rust
-      - name: Test Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
 
   test_ubuntu_clang:
     name: Ubuntu (Clang 16) - ${{ matrix.os }}
@@ -240,9 +238,10 @@ jobs:
           cmake -B build_shared -DCMAKE_BUILD_TYPE=RelWithDebInfo -DSTRINGZILLA_BUILD_SHARED=1
           cmake --build build_shared --target stringzilla_shared --config RelWithDebInfo
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version: ">=1.24"
+          cache: false
       - name: Test Go binding
         working-directory: golang
         env:
@@ -283,11 +282,8 @@ jobs:
         run: python -X faulthandler -m pytest scripts/test_stringzilla.py -s -vv --maxfail=1 --full-trace
 
         # Rust
-      - name: Test Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
 
         # Swift
         # Fails due to: https://github.com/swift-actions/setup-swift/issues/591
@@ -517,7 +513,7 @@ jobs:
             target_arch: x86-64
             target: x86_64-linux-gnu
             target_processor: amd64
-    
+
     env:
       CC: clang-16
       CXX: clang++-16
@@ -550,7 +546,7 @@ jobs:
           sudo ./llvm.sh 16
 
       - name: Build C/C++ for ${{ matrix.target_arch }}
-        run: |          
+        run: |
           # Configure and build the project
           cmake -B build_artifacts \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
@@ -647,11 +643,8 @@ jobs:
         run: swift test
 
         # Rust
-      - name: Test Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
 
   test_windows:
     name: Windows - ${{ matrix.os }}
@@ -734,9 +727,10 @@ jobs:
       image: alpine:latest
       options: --privileged # If needed for certain Docker operations
     steps:
+      # bash and curl are required by dtolnay/rust-toolchain
       - name: Install Git for checkout
         run: |
-          apk add --no-cache git openssh ca-certificates
+          apk add --no-cache git openssh ca-certificates bash curl
           git --version
       - name: Log Alpine and Git versions
         run: |
@@ -782,11 +776,8 @@ jobs:
         run: pytest scripts/test_stringzilla.py -s -x
 
         # Rust
-      - name: Test Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
 
   build_wheels:
     name: Build Python ${{ matrix.python-version }} for ${{ matrix.os }}

.github/workflows/release.yml

@@ -534,10 +534,8 @@ jobs:
       - name: Hide submodule Crates
         run: |
           mv fork_union/Cargo.toml fork_union/Cargo.toml.bak
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - name: Set up Rust
+        uses: dtolnay/rust-toolchain@stable
       - uses: katyo/publish-crates@v2
         with:
           registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

.gitignore

@@ -1,17 +1,11 @@
-# Primary build folders
-build/
-build_debug/
-build_release/
-build_relwithdebinfo/
-build_go/
-build_golang/
-build_test/
-build_artifacts*
+# Primary root-level build folders
+/build*/
 
 # Yes, everyone loves keeping this file in the history.
-# But with a very minimalistic binding and just a couple of dependencies 
+# But with a very minimalistic binding and just a couple of dependencies
 # it brings 7000 lines of text polluting the entire repo.
 package-lock.json
+uv.lock
 
 # Temporary files
 .DS_Store
@@ -34,6 +28,7 @@ CMakeFiles
 node_modules/
 .vs/
 *.tar.gz
+.tmp/
 
 # Recommended datasets
 utf8.txt
@@ -49,4 +44,4 @@ acgt_1m.txt
 
 # StringZilla-specific log files
 /failed_sz_*
-/.cache/
+/.cache/

.vscode/extensions.json

@@ -4,8 +4,9 @@
     "ms-vscode.cmake-tools",
     "ms-python.python",
     "ms-python.black-formatter",
+    "esbenp.prettier-vscode",
     "yzhang.markdown-all-in-one",
     "aaron-bond.better-comments",
     "cheshirekow.cmake-format"
   ]
-}
+}

.vscode/launch.json

@@ -203,16 +203,22 @@
       "program": "${file}",
       "console": "integratedTerminal",
       "justMyCode": true,
-      "args": ["./leipzig1M.txt"]
+      "args": [
+        "./leipzig1M.txt"
+      ]
     },
     {
       "name": "Current PyTest File",
       "type": "debugpy",
       "request": "launch",
       "module": "pytest",
-      "args": ["${file}", "-s", "-x"],
+      "args": [
+        "${file}",
+        "-s",
+        "-x"
+      ],
       "console": "integratedTerminal",
       "justMyCode": false
     }
   ]
-}
+}

.vscode/settings.json

@@ -13,6 +13,16 @@
     "editor.insertSpaces": true,
     "editor.tabSize": 2
   },
+  "[jsonc]": {
+    "editor.defaultFormatter": "vscode.json-language-features",
+    "editor.insertSpaces": true,
+    "editor.tabSize": 2
+  },
+  "[python]": {
+    "editor.defaultFormatter": "ms-python.black-formatter",
+    "editor.insertSpaces": true,
+    "editor.tabSize": 4
+  },
   "[typescript]": {
     "editor.defaultFormatter": "vscode.typescript-language-features",
     "editor.insertSpaces": true,
@@ -261,24 +271,10 @@
   "editor.detectIndentation": false,
   "editor.formatOnSave": true,
   "editor.insertSpaces": true,
-  "editor.rulers": [120],
+  "editor.rulers": [
+    120
+  ],
   "editor.tabSize": 4,
-  // Rust analyzer memory optimizations
-  "rust-analyzer.checkOnSave.enable": true,
-  "rust-analyzer.checkOnSave.allTargets": false,
-  "rust-analyzer.cargo.allFeatures": false,
-  "rust-analyzer.cargo.buildScripts.enable": false,
-  "rust-analyzer.procMacro.enable": false,
-  "rust-analyzer.diagnostics.enable": true,
-  "rust-analyzer.diagnostics.experimental.enable": false,
-  "rust-analyzer.completion.autoimport.enable": false,
-  "rust-analyzer.lens.enable": false,
-  "rust-analyzer.inlayHints.enable": false,
-  "rust-analyzer.hover.actions.enable": false,
-  "rust-analyzer.assist.emitMustUse": false,
-  "rust-analyzer.cachePriming.enable": false,
-  "rust-analyzer.files.watcher": "client",
-  "rust-analyzer.updates.channel": "stable",
   "files.associations": {
     "__availability": "cpp",
     "__bit_reference": "cpp",
@@ -395,6 +391,7 @@
     "types.h": "c",
     "unordered_map": "cpp",
     "unordered_set": "cpp",
+    "utf8_unpack.h": "c",
     "utility": "cpp",
     "variant": "cpp",
     "vector": "cpp",
@@ -414,5 +411,22 @@
     "xtr1common": "cpp",
     "xtree": "cpp",
     "xutility": "cpp"
-  }
-}
+  },
+  "notebook.formatOnSave.enabled": true,
+  "rust-analyzer.assist.emitMustUse": false,
+  "rust-analyzer.cachePriming.enable": false,
+  "rust-analyzer.cargo.allFeatures": false,
+  "rust-analyzer.cargo.buildScripts.enable": false,
+  "rust-analyzer.checkOnSave.allTargets": false,
+  // Rust analyzer memory optimizations
+  "rust-analyzer.checkOnSave.enable": true,
+  "rust-analyzer.completion.autoimport.enable": false,
+  "rust-analyzer.diagnostics.enable": true,
+  "rust-analyzer.diagnostics.experimental.enable": false,
+  "rust-analyzer.files.watcher": "client",
+  "rust-analyzer.hover.actions.enable": false,
+  "rust-analyzer.inlayHints.enable": false,
+  "rust-analyzer.lens.enable": false,
+  "rust-analyzer.procMacro.enable": false,
+  "rust-analyzer.updates.channel": "stable"
+}

CONTRIBUTING.md

@@ -180,6 +180,36 @@ build_debug/stringzilla_test_cpp20_serial     # Arm variant compiled without Neo
 
 Note, that Address Sanitizers have a hard time with masked load and store instructions in AVX-512 and SVE.
 
+The C++ and Python test suites support environment variables for reproducible stress testing and CI fuzzing:
+
+| Variable              | Description                                         | Default |
+| :-------------------- | :-------------------------------------------------- | ------: |
+| `SZ_TESTS_SEED`       | Seed for the random number generator                |  Random |
+| `SZ_TESTS_MULTIPLIER` | Scales all baseline iteration counts proportionally |     1.0 |
+
+Each test has its own baseline iteration count tuned for its operation complexity.
+The multiplier scales all baselines proportionally - use `0.1` for quick smoke tests or `10` for thorough stress testing.
+
+```bash
+# Run with a specific seed for reproducibility
+SZ_TESTS_SEED=42 build_debug/stringzilla_test_cpp20
+
+# Quick smoke test (10% of normal iterations)
+SZ_TESTS_MULTIPLIER=0.1 build_debug/stringzilla_test_cpp20
+
+# Thorough CI stress test (10x normal iterations)
+SZ_TESTS_MULTIPLIER=10 build_debug/stringzilla_test_cpp20
+
+# Combine both for CI fuzzing
+SZ_TESTS_SEED=12345 SZ_TESTS_MULTIPLIER=5 build_debug/stringzilla_test_cpp20
+
+# Python tests also respect SZ_TESTS_SEED
+SZ_TESTS_SEED=42 pytest scripts/test_stringzilla.py -v
+```
+
+When a test fails, note the seed from the output and re-run with that exact seed to reproduce the issue.
+This is particularly useful for debugging SIMD edge cases that only manifest with specific input patterns.
+
 To use CppCheck for static analysis make sure to export the compilation commands.
 Overall, CppCheck and Clang-Tidy are extremely noisy and not suitable for CI, but may be useful for local development.