Add support for Split pretokenizer w/ behavior=removed & invert=false

Author: xenova

.github/workflows/format-check.yml

@@ -0,0 +1,29 @@
+name: Prettier Check
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  format:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        node-version: [22]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: format:check
+        run: npm run format:check

.prettierignore

@@ -6,3 +6,7 @@ examples
 scripts
 types
 *.md
+# Ignore python related files
+.venv
+# Mac files
+.DS_STORE

.prettierrc

@@ -1,9 +1,11 @@
 {
+    "tabWidth": 4,
+    "printWidth": 120,
     "overrides": [
         {
             "files": ["tests/**/*.js"],
             "options": {
-              "printWidth": 10000000
+                "printWidth": 10000000
             }
         }
     ]

jest.config.mjs

@@ -4,188 +4,185 @@
  */
 
 export default {
-  // All imported modules in your tests should be mocked automatically
-  // automock: false,
+    // All imported modules in your tests should be mocked automatically
+    // automock: false,
 
-  // Stop running tests after `n` failures
-  // bail: 0,
+    // Stop running tests after `n` failures
+    // bail: 0,
 
-  // Automatically clear mock calls, instances, contexts and results before every test
-  clearMocks: true,
+    // Automatically clear mock calls, instances, contexts and results before every test
+    clearMocks: true,
 
-  // Indicates whether the coverage information should be collected while executing the test
-  collectCoverage: true,
+    // Indicates whether the coverage information should be collected while executing the test
+    collectCoverage: true,
 
-  // An array of glob patterns indicating a set of files for which coverage information should be collected
-  // collectCoverageFrom: undefined,
+    // An array of glob patterns indicating a set of files for which coverage information should be collected
+    // collectCoverageFrom: undefined,
 
-  // The directory where Jest should output its coverage files
-  coverageDirectory: "coverage",
+    // The directory where Jest should output its coverage files
+    coverageDirectory: "coverage",
 
-  // An array of regexp pattern strings used to skip coverage collection
-  coveragePathIgnorePatterns: [
-    "node_modules",
-    "tests",
-  ],
+    // An array of regexp pattern strings used to skip coverage collection
+    coveragePathIgnorePatterns: ["node_modules", "tests"],
 
-  // Indicates which provider should be used to instrument code for coverage
-  coverageProvider: "v8",
+    // Indicates which provider should be used to instrument code for coverage
+    coverageProvider: "v8",
 
-  // A list of reporter names that Jest uses when writing coverage reports
-  // coverageReporters: [
-  //   "json",
-  //   "text",
-  //   "lcov",
-  //   "clover"
-  // ],
+    // A list of reporter names that Jest uses when writing coverage reports
+    // coverageReporters: [
+    //   "json",
+    //   "text",
+    //   "lcov",
+    //   "clover"
+    // ],
 
-  // An object that configures minimum threshold enforcement for coverage results
-  // coverageThreshold: undefined,
+    // An object that configures minimum threshold enforcement for coverage results
+    // coverageThreshold: undefined,
 
-  // A path to a custom dependency extractor
-  // dependencyExtractor: undefined,
+    // A path to a custom dependency extractor
+    // dependencyExtractor: undefined,
 
-  // Make calling deprecated APIs throw helpful error messages
-  // errorOnDeprecated: false,
+    // Make calling deprecated APIs throw helpful error messages
+    // errorOnDeprecated: false,
 
-  // The default configuration for fake timers
-  // fakeTimers: {
-  //   "enableGlobally": false
-  // },
+    // The default configuration for fake timers
+    // fakeTimers: {
+    //   "enableGlobally": false
+    // },
 
-  // Force coverage collection from ignored files using an array of glob patterns
-  // forceCoverageMatch: [],
+    // Force coverage collection from ignored files using an array of glob patterns
+    // forceCoverageMatch: [],
 
-  // A path to a module which exports an async function that is triggered once before all test suites
-  // globalSetup: undefined,
+    // A path to a module which exports an async function that is triggered once before all test suites
+    // globalSetup: undefined,
 
-  // A path to a module which exports an async function that is triggered once after all test suites
-  // globalTeardown: undefined,
+    // A path to a module which exports an async function that is triggered once after all test suites
+    // globalTeardown: undefined,
 
-  // A set of global variables that need to be available in all test environments
-  // globals: {},
+    // A set of global variables that need to be available in all test environments
+    // globals: {},
 
-  // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
-  // maxWorkers: "50%",
+    // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+    // maxWorkers: "50%",
 
-  // An array of directory names to be searched recursively up from the requiring module's location
-  // moduleDirectories: [
-  //   "node_modules"
-  // ],
+    // An array of directory names to be searched recursively up from the requiring module's location
+    // moduleDirectories: [
+    //   "node_modules"
+    // ],
 
-  // An array of file extensions your modules use
-  // moduleFileExtensions: [
-  //   "js",
-  //   "mjs",
-  //   "cjs",
-  //   "jsx",
-  //   "ts",
-  //   "tsx",
-  //   "json",
-  //   "node"
-  // ],
+    // An array of file extensions your modules use
+    // moduleFileExtensions: [
+    //   "js",
+    //   "mjs",
+    //   "cjs",
+    //   "jsx",
+    //   "ts",
+    //   "tsx",
+    //   "json",
+    //   "node"
+    // ],
 
-  // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
-  // moduleNameMapper: {},
+    // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module
+    // moduleNameMapper: {},
 
-  // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
-  // modulePathIgnorePatterns: [],
+    // An array of regexp pattern strings, matched against all module paths before considered 'visible' to the module loader
+    // modulePathIgnorePatterns: [],
 
-  // Activates notifications for test results
-  // notify: false,
+    // Activates notifications for test results
+    // notify: false,
 
-  // An enum that specifies notification mode. Requires { notify: true }
-  // notifyMode: "failure-change",
+    // An enum that specifies notification mode. Requires { notify: true }
+    // notifyMode: "failure-change",
 
-  // A preset that is used as a base for Jest's configuration
-  // preset: undefined,
+    // A preset that is used as a base for Jest's configuration
+    // preset: undefined,
 
-  // Run tests from one or more projects
-  // projects: undefined,
+    // Run tests from one or more projects
+    // projects: undefined,
 
-  // Use this configuration option to add custom reporters to Jest
-  // reporters: undefined,
+    // Use this configuration option to add custom reporters to Jest
+    // reporters: undefined,
 
-  // Automatically reset mock state before every test
-  // resetMocks: false,
+    // Automatically reset mock state before every test
+    // resetMocks: false,
 
-  // Reset the module registry before running each individual test
-  // resetModules: false,
+    // Reset the module registry before running each individual test
+    // resetModules: false,
 
-  // A path to a custom resolver
-  // resolver: undefined,
+    // A path to a custom resolver
+    // resolver: undefined,
 
-  // Automatically restore mock state and implementation before every test
-  // restoreMocks: false,
+    // Automatically restore mock state and implementation before every test
+    // restoreMocks: false,
 
-  // The root directory that Jest should scan for tests and modules within
-  // rootDir: undefined,
+    // The root directory that Jest should scan for tests and modules within
+    // rootDir: undefined,
 
-  // A list of paths to directories that Jest should use to search for files in
-  roots: ["./tests/"],
+    // A list of paths to directories that Jest should use to search for files in
+    roots: ["./tests/"],
 
-  // Allows you to use a custom runner instead of Jest's default test runner
-  // runner: "jest-runner",
+    // Allows you to use a custom runner instead of Jest's default test runner
+    // runner: "jest-runner",
 
-  // The paths to modules that run some code to configure or set up the testing environment before each test
-  // setupFiles: [],
+    // The paths to modules that run some code to configure or set up the testing environment before each test
+    // setupFiles: [],
 
-  // A list of paths to modules that run some code to configure or set up the testing framework before each test
-  // setupFilesAfterEnv: [],
+    // A list of paths to modules that run some code to configure or set up the testing framework before each test
+    // setupFilesAfterEnv: [],
 
-  // The number of seconds after which a test is considered as slow and reported as such in the results.
-  // slowTestThreshold: 5,
+    // The number of seconds after which a test is considered as slow and reported as such in the results.
+    // slowTestThreshold: 5,
 
-  // A list of paths to snapshot serializer modules Jest should use for snapshot testing
-  // snapshotSerializers: [],
+    // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+    // snapshotSerializers: [],
 
-  // The test environment that will be used for testing
-  // testEnvironment: "jest-environment-node",
+    // The test environment that will be used for testing
+    // testEnvironment: "jest-environment-node",
 
-  // Options that will be passed to the testEnvironment
-  // testEnvironmentOptions: {},
+    // Options that will be passed to the testEnvironment
+    // testEnvironmentOptions: {},
 
-  // Adds a location field to test results
-  // testLocationInResults: false,
+    // Adds a location field to test results
+    // testLocationInResults: false,
 
-  // The glob patterns Jest uses to detect test files
-  // testMatch: [
-  //   "**/__tests__/**/*.[jt]s?(x)",
-  //   "**/?(*.)+(spec|test).[tj]s?(x)"
-  // ],
+    // The glob patterns Jest uses to detect test files
+    // testMatch: [
+    //   "**/__tests__/**/*.[jt]s?(x)",
+    //   "**/?(*.)+(spec|test).[tj]s?(x)"
+    // ],
 
-  // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
-  // testPathIgnorePatterns: [
-  //   "\\\\node_modules\\\\"
-  // ],
+    // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+    // testPathIgnorePatterns: [
+    //   "\\\\node_modules\\\\"
+    // ],
 
-  // The regexp pattern or array of patterns that Jest uses to detect test files
-  // testRegex: [],
+    // The regexp pattern or array of patterns that Jest uses to detect test files
+    // testRegex: [],
 
-  // This option allows the use of a custom results processor
-  // testResultsProcessor: undefined,
+    // This option allows the use of a custom results processor
+    // testResultsProcessor: undefined,
 
-  // This option allows use of a custom test runner
-  // testRunner: "jest-circus/runner",
+    // This option allows use of a custom test runner
+    // testRunner: "jest-circus/runner",
 
-  // A map from regular expressions to paths to transformers
-  transform: {},
+    // A map from regular expressions to paths to transformers
+    transform: {},
 
-  // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
-  // transformIgnorePatterns: [
-  //   "\\\\node_modules\\\\",
-  //   "\\.pnp\\.[^\\\\]+$"
-  // ],
+    // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+    // transformIgnorePatterns: [
+    //   "\\\\node_modules\\\\",
+    //   "\\.pnp\\.[^\\\\]+$"
+    // ],
 
-  // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
-  // unmockedModulePathPatterns: undefined,
+    // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+    // unmockedModulePathPatterns: undefined,
 
-  // Indicates whether each individual test should be reported during the run
-  // verbose: undefined,
+    // Indicates whether each individual test should be reported during the run
+    // verbose: undefined,
 
-  // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
-  // watchPathIgnorePatterns: [],
+    // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+    // watchPathIgnorePatterns: [],
 
-  // Whether to use watchman for file crawling
-  // watchman: true,
+    // Whether to use watchman for file crawling
+    // watchman: true,
 };

jsconfig.json

@@ -1,14 +1,14 @@
 {
-  // Only include files in the src directory
-  "include": ["src/**/*"],
-  "compilerOptions": {
-    // Tells the compiler to check JS files
-    "checkJs": true,
-    "target": "esnext",
-    "module": "nodenext",
-    "moduleResolution": "nodenext"
-  },
-  "typeAcquisition": {
-    "include": ["jest"]
-  }
+    // Only include files in the src directory
+    "include": ["src/**/*"],
+    "compilerOptions": {
+        // Tells the compiler to check JS files
+        "checkJs": true,
+        "target": "esnext",
+        "module": "nodenext",
+        "moduleResolution": "nodenext"
+    },
+    "typeAcquisition": {
+        "include": ["jest"]
+    }
 }