Upgrade Transformers.js to v4

Move the local embedding and reranker integrations onto the stable v4 API and ONNX Runtime 1.24.3. This removes the macOS crash dependency while preserving the tokenizer pair fix already covered by this branch.

Author: aolin480

package.json

@@ -128,8 +128,8 @@
     "prepare": "husky"
   },
   "dependencies": {
+    "@huggingface/transformers": "^4.2.0",
     "@inquirer/prompts": "^3.0.0",
-    "@huggingface/transformers": "^3.8.1",
     "@lancedb/lancedb": "^0.4.0",
     "@modelcontextprotocol/sdk": "^1.27.1",
     "@typescript-eslint/typescript-estree": "^7.0.0",
@@ -172,7 +172,7 @@
       "@modelcontextprotocol/sdk>ajv": "8.18.0",
       "@modelcontextprotocol/sdk>@hono/node-server": "1.19.13",
       "@modelcontextprotocol/sdk>express-rate-limit": "8.2.2",
-      "@huggingface/transformers>onnxruntime-node": "1.24.2",
+      "@huggingface/transformers>onnxruntime-node": "1.24.3",
       "path-to-regexp": "8.4.0",
       "brace-expansion": "5.0.5",
       "micromatch>picomatch": "2.3.2",

pnpm-lock.yaml

@@ -9,6 +9,7 @@
  */
 
 import type { SearchResult } from '../types/index.js';
+import type { PreTrainedTokenizer } from '@huggingface/transformers';
 import os from 'os';
 
 const DEFAULT_RERANKER_MODEL = 'Xenova/ms-marco-MiniLM-L-6-v2';
@@ -19,25 +20,11 @@ const RERANK_TOP_K = 10;
 /** Trigger reranking when the score gap between #1 and #3 is below this threshold */
 const AMBIGUITY_THRESHOLD = 0.08;
 
-interface CrossEncoderTokenizer {
-  (
-    text: string,
-    options?: {
-      text_pair?: string | null;
-      padding?: boolean | 'max_length';
-      truncation?: boolean | null;
-      max_length?: number | null;
-      return_tensor?: boolean;
-      return_token_type_ids?: boolean | null;
-    }
-  ): unknown;
-}
-
 interface CrossEncoderModel {
   (inputs: unknown): Promise<{ logits: { data: ArrayLike<number> } }>;
 }
 
-let cachedTokenizer: CrossEncoderTokenizer | null = null;
+let cachedTokenizer: PreTrainedTokenizer | null = null;
 let cachedModel: CrossEncoderModel | null = null;
 let initPromise: Promise<void> | null = null;
 

src/core/reranker.ts

@@ -1,5 +1,5 @@
 import { EmbeddingProvider, DEFAULT_MODEL } from './types.js';
-import type { FeatureExtractionPipelineType } from '@huggingface/transformers';
+import type { FeatureExtractionPipeline } from '@huggingface/transformers';
 import os from 'os';
 
 /**
@@ -43,7 +43,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
   readonly modelName: string;
   readonly dimensions: number;
 
-  private pipeline: FeatureExtractionPipelineType | null = null;
+  private pipeline: FeatureExtractionPipeline | null = null;
   private ready = false;
   private initPromise: Promise<void> | null = null;
 
@@ -75,7 +75,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
         task: 'feature-extraction',
         model: string,
         opts: Record<string, unknown>
-      ) => Promise<FeatureExtractionPipelineType>;
+      ) => Promise<FeatureExtractionPipeline>;
       this.pipeline = await (pipeline as PipelineFn)('feature-extraction', this.modelName, {
         dtype: 'q8',
         // Limit ONNX Runtime to half cores by default — prevents system freeze during indexing.