fix: singleton embedding provider and LanceDB schema validation

- Implement module-level singleton for TransformersEmbeddingProvider to prevent duplicate model loads
- Add getEmbeddingProvider() factory function with lazy initialization
- Add schema validation in LanceDB to detect and auto-drop stale tables missing vector column
- Update logging to use MCP server.sendLoggingMessage instead of console.error

Author: PatrickSys

src/embeddings/index.ts

@@ -1,44 +1,48 @@
-/**
- * Embeddings module
- * Provides local embedding generation using Transformers.js
- */
-
 export * from './types.js';
 export * from './transformers.js';
 
 import { EmbeddingProvider, EmbeddingConfig, DEFAULT_EMBEDDING_CONFIG } from './types.js';
 import { TransformersEmbeddingProvider } from './transformers.js';
 
-/**
- * Get an embedding provider based on configuration
- */
+let cachedProvider: EmbeddingProvider | null = null;
+let cachedProviderType: string | null = null;
+
 export async function getEmbeddingProvider(
   config: Partial<EmbeddingConfig> = {}
 ): Promise<EmbeddingProvider> {
   const mergedConfig = { ...DEFAULT_EMBEDDING_CONFIG, ...config };
+  const providerKey = `${mergedConfig.provider}:${mergedConfig.model}`;
+
+  if (cachedProvider && cachedProviderType === providerKey) {
+    return cachedProvider;
+  }
 
   if (mergedConfig.provider === 'openai') {
     const { OpenAIEmbeddingProvider } = await import('./openai.js');
     const provider = new OpenAIEmbeddingProvider(
-      mergedConfig.model === 'Xenova/bge-small-en-v1.5' ? 'text-embedding-3-small' : mergedConfig.model,
+      mergedConfig.model || 'text-embedding-3-small',
       mergedConfig.apiKey,
       mergedConfig.apiEndpoint
     );
     await provider.initialize();
+    cachedProvider = provider;
+    cachedProviderType = providerKey;
     return provider;
   }
 
   if (mergedConfig.provider === 'custom') {
-    throw new Error("Custom provider requires implementing 'IEmbeddingProvider' and bundling it. Use 'openai' or 'transformers' for now.");
+    throw new Error("Custom provider not implemented. Use 'openai' or 'transformers'.");
   }
 
-  // Ollama support can be added later
   if (mergedConfig.provider === 'ollama') {
     console.warn('Ollama provider not yet implemented, falling back to Transformers.js');
   }
 
   const provider = new TransformersEmbeddingProvider(mergedConfig.model);
   await provider.initialize();
+  cachedProvider = provider;
+  cachedProviderType = providerKey;
 
   return provider;
 }
+

src/embeddings/transformers.ts

@@ -1,11 +1,5 @@
-/**
- * Transformers.js Embedding Provider
- * Uses local models via @xenova/transformers
- */
+import { EmbeddingProvider, DEFAULT_MODEL } from "./types.js";
 
-import { EmbeddingProvider } from "./types.js";
-
-// Model configurations
 const MODEL_CONFIGS: Record<string, { dimensions: number }> = {
   "Xenova/bge-small-en-v1.5": { dimensions: 384 },
   "Xenova/all-MiniLM-L6-v2": { dimensions: 384 },
@@ -21,7 +15,7 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
   private ready = false;
   private initPromise: Promise<void> | null = null;
 
-  constructor(modelName: string = "Xenova/bge-small-en-v1.5") {
+  constructor(modelName: string = DEFAULT_MODEL) {
     this.modelName = modelName;
     this.dimensions = MODEL_CONFIGS[modelName]?.dimensions || 384;
   }
@@ -39,12 +33,10 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
       console.error(`Loading embedding model: ${this.modelName}`);
       console.error("(First run will download ~130MB model)");
 
-      // Dynamic import to avoid issues at require time
       const { pipeline } = await import("@xenova/transformers");
 
-      // Create feature extraction pipeline
       this.pipeline = await pipeline("feature-extraction", this.modelName, {
-        quantized: true, // Use quantized model for speed
+        quantized: true,
       });
 
       this.ready = true;
@@ -61,13 +53,11 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
     }
 
     try {
-      // Get embeddings
       const output = await this.pipeline(text, {
         pooling: "mean",
         normalize: true,
       });
 
-      // Convert to array
       return Array.from(output.data);
     } catch (error) {
       console.error("Failed to generate embedding:", error);
@@ -81,24 +71,19 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
     }
 
     const embeddings: number[][] = [];
-
-    // Process in smaller batches to manage memory
     const batchSize = 32;
+
     for (let i = 0; i < texts.length; i += batchSize) {
       const batch = texts.slice(i, i + batchSize);
-
-      // Process batch
       const batchEmbeddings = await Promise.all(
         batch.map((text) => this.embed(text))
       );
 
       embeddings.push(...batchEmbeddings);
 
-      // Log progress for large batches
       if (texts.length > 100 && (i + batchSize) % 100 === 0) {
         console.error(
-          `Embedded ${Math.min(i + batchSize, texts.length)}/${texts.length
-          } chunks`
+          `Embedded ${Math.min(i + batchSize, texts.length)}/${texts.length} chunks`
         );
       }
     }
@@ -111,13 +96,11 @@ export class TransformersEmbeddingProvider implements EmbeddingProvider {
   }
 }
 
-/**
- * Create an embedding provider based on config
- */
 export async function createEmbeddingProvider(
-  modelName: string = "Xenova/bge-base-en-v1.5"
+  modelName: string = DEFAULT_MODEL
 ): Promise<EmbeddingProvider> {
   const provider = new TransformersEmbeddingProvider(modelName);
   await provider.initialize();
   return provider;
 }
+

src/embeddings/types.ts

@@ -1,30 +1,11 @@
-/**
- * Types for embedding providers
- */
-
 export interface EmbeddingProvider {
   readonly name: string;
   readonly modelName: string;
   readonly dimensions: number;
 
-  /**
-   * Initialize the provider (load model, etc.)
-   */
   initialize(): Promise<void>;
-
-  /**
-   * Generate embedding for a single text
-   */
   embed(text: string): Promise<number[]>;
-
-  /**
-   * Generate embeddings for multiple texts (batch)
-   */
   embedBatch(texts: string[]): Promise<number[][]>;
-
-  /**
-   * Check if provider is ready
-   */
   isReady(): boolean;
 }
 
@@ -37,10 +18,13 @@ export interface EmbeddingConfig {
   apiEndpoint?: string;
 }
 
+export const DEFAULT_MODEL = process.env.EMBEDDING_MODEL || "Xenova/bge-small-en-v1.5";
+
 export const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig = {
   provider: (process.env.EMBEDDING_PROVIDER as any) || "transformers",
-  model: process.env.EMBEDDING_MODEL || "Xenova/bge-small-en-v1.5",
+  model: DEFAULT_MODEL,
   batchSize: 32,
   maxRetries: 3,
   apiKey: process.env.OPENAI_API_KEY,
 };
+

src/index.ts

@@ -75,6 +75,7 @@ const server = new Server(
     capabilities: {
       tools: {},
       resources: {},
+      logging: {}, // Enable structured logging for clients that support it
     },
   }
 );

src/storage/lancedb.ts

@@ -35,11 +35,29 @@ export class LanceDBStorageProvider implements VectorStorageProvider {
       // Connect to database
       this.db = await lancedb.connect(storagePath);
 
-      // Check if table exists
+      // Check if table exists and has valid schema
       const tableNames = await this.db.tableNames();
       if (tableNames.includes('code_chunks')) {
         this.table = await this.db.openTable('code_chunks');
-        console.error('Opened existing LanceDB table');
+
+        // Validate schema has vector column (required for semantic search)
+        try {
+          const schema = await this.table.schema();
+          const hasVectorColumn = schema.fields.some((f: any) => f.name === 'vector');
+
+          if (!hasVectorColumn) {
+            console.error('Stale index detected (missing vector column). Rebuilding...');
+            await this.db.dropTable('code_chunks');
+            this.table = null;
+          } else {
+            console.error('Opened existing LanceDB table');
+          }
+        } catch (schemaError) {
+          // If schema check fails, table is likely corrupted - drop and rebuild
+          console.error('Failed to validate table schema, rebuilding index...');
+          await this.db.dropTable('code_chunks');
+          this.table = null;
+        }
       }
 
       this.initialized = true;