fix(providers): improve Python provider reliability and logging (#6034)

Author: mldangelo

CHANGELOG.md

@@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
 ### Fixed
 
 - fix(providers): revert eager template rendering that broke runtime variable substitution (5423f80)
+- fix(providers): improve Python provider reliability with automatic python3/python detection, worker cleanup, request count tracking, and reduced logging noise (#6034)
 - fix(providers): simulated-user and mischievous-user now respect assistant system prompts in multi-turn conversations (#6020)
 - fix(providers): improve MCP tool schema transformation for OpenAI compatibility (#5965)
 - fix(providers): sessionId now properly stored in metadata for providers that use server side generated sessionIds (#6016)

src/evaluator.ts

@@ -20,6 +20,7 @@ import { selectMaxScore } from './matchers';
 import { generateIdFromPrompt } from './models/prompt';
 import { CIProgressReporter } from './progress/ciProgressReporter';
 import { maybeEmitAzureOpenAiWarning } from './providers/azure/warnings';
+import { providerRegistry } from './providers/providerRegistry';
 import { isPromptfooSampleTarget } from './providers/shared';
 import { strategyDisplayNames } from './redteam/constants';
 import { getSessionId, isSimbaTestCase } from './redteam/util';
@@ -1911,6 +1912,9 @@ class Evaluator {
         await sleep(1000);
       }
       await stopOtlpReceiverIfNeeded();
+
+      // Clean up Python worker pools to prevent resource leaks
+      await providerRegistry.shutdownAll();
     }
   }
 }

src/providers/providerRegistry.ts

@@ -57,7 +57,15 @@ class ProviderRegistry {
   }
 
   async shutdownAll(): Promise<void> {
-    await Promise.all(Array.from(this.providers).map((p) => p.shutdown()));
+    const results = await Promise.allSettled(Array.from(this.providers).map((p) => p.shutdown()));
+
+    // Log any failures but don't throw - cleanup should be defensive
+    for (const result of results) {
+      if (result.status === 'rejected') {
+        logger.warn(`Error shutting down provider: ${result.reason}`);
+      }
+    }
+
     this.providers.clear();
   }
 }

src/providers/pythonCompletion.ts

@@ -199,6 +199,7 @@ export class PythonProvider implements ApiProvider {
           parsedResult.tokenUsage = {
             cached: total,
             total,
+            numRequests: parsedResult.tokenUsage.numRequests ?? 1,
           };
           logger.debug(
             `Updated token usage for cached result: ${JSON.stringify(parsedResult.tokenUsage)}`,
@@ -310,10 +311,18 @@ export class PythonProvider implements ApiProvider {
         );
       }
 
-      // Set cached=false on fresh results
+      // Set cached=false on fresh results and ensure numRequests is set
       if (typeof result === 'object' && result !== null && apiType === 'call_api') {
         logger.debug(`PythonProvider explicitly setting cached=false for fresh result`);
         result.cached = false;
+
+        // Ensure tokenUsage includes numRequests for fresh results
+        if (result.tokenUsage && !result.tokenUsage.numRequests) {
+          result.tokenUsage.numRequests = 1;
+          logger.debug(
+            `Added numRequests to fresh result token usage: ${JSON.stringify(result.tokenUsage)}`,
+          );
+        }
       }
 
       return result;

src/python/persistent_wrapper.py

@@ -30,12 +30,6 @@ def load_user_module(script_path):
     if script_dir not in sys.path:
         sys.path.insert(0, script_dir)
 
-    print(
-        f"Loading module {module_name} from {script_path}...",
-        file=sys.stderr,
-        flush=True,
-    )
-
     spec = importlib.util.spec_from_file_location(module_name, script_path)
     if spec is None or spec.loader is None:
         raise ImportError(f"Cannot load module from {script_path}")

src/python/worker.ts

@@ -4,6 +4,7 @@ import path from 'path';
 import os from 'os';
 import logger from '../logger';
 import { safeJsonStringify } from '../util/json';
+import { validatePythonPath } from './pythonUtils';
 
 export class PythonWorker {
   private process: PythonShell | null = null;
@@ -33,9 +34,15 @@ export class PythonWorker {
   private async startWorker(): Promise<void> {
     const wrapperPath = path.join(__dirname, 'persistent_wrapper.py');
 
+    // Validate and resolve Python path using smart detection (tries python3, then python)
+    const resolvedPythonPath = await validatePythonPath(
+      this.pythonPath || 'python',
+      typeof this.pythonPath === 'string',
+    );
+
     this.process = new PythonShell(wrapperPath, {
       mode: 'text',
-      pythonPath: this.pythonPath || 'python',
+      pythonPath: resolvedPythonPath,
       args: [this.scriptPath, this.functionName],
       stdio: ['pipe', 'pipe', 'pipe'],
     });

test/providers/pythonCompletion.test.ts

@@ -286,6 +286,7 @@ describe('PythonProvider', () => {
         tokenUsage: {
           cached: 25,
           total: 25,
+          numRequests: 1,
         },
       });
     });
@@ -316,6 +317,7 @@ describe('PythonProvider', () => {
           prompt: 12,
           completion: 18,
           total: 30,
+          numRequests: 1,
         },
       });
     });
@@ -364,6 +366,7 @@ describe('PythonProvider', () => {
       expect(result.tokenUsage).toEqual({
         cached: 0,
         total: 0,
+        numRequests: 1,
       });
       expect(result.cached).toBe(true);
     });