fix bt eval with monorepo and without extensions in imports

scripts/eval-runner.ts

@@ -191,6 +191,23 @@ function readRunnerConfig(): RunnerConfig {
   };
 }
 
+function readEvalFiles(): string[] {
+  const raw = process.env.BT_EVAL_FILES;
+  if (raw) {
+    const parsed = JSON.parse(raw);
+    if (
+      !Array.isArray(parsed) ||
+      !parsed.every((f: unknown) => typeof f === "string")
+    ) {
+      throw new Error("BT_EVAL_FILES must be a JSON array of strings.");
+    }
+    return parsed;
+  }
+  // Fallback: read from argv for backwards compatibility (e.g., running
+  // the eval-runner directly outside of bt).
+  return process.argv.slice(2);
+}
+
 const runtimeRequire = createRequire(
   process.argv[1] ?? path.join(process.cwd(), "package.json"),
 );
@@ -396,6 +413,22 @@ function resolveLocalSpecifier(
   return null;
 }
 
+const TS_RESOLVE_EXTENSIONS = [".ts", ".tsx", ".mts", ".cts"];
+
+function isRelativeOrAbsoluteSpecifier(specifier: string): boolean {
+  return (
+    specifier.startsWith("./") ||
+    specifier.startsWith("../") ||
+    specifier.startsWith("/") ||
+    specifier.startsWith("file://")
+  );
+}
+
+function specifierHasExtension(specifier: string): boolean {
+  const lastSegment = specifier.split("/").pop() ?? specifier;
+  return lastSegment.includes(".");
+}
+
 function installNodeModuleHooks() {
   const registerHooks = moduleMutable.registerHooks as
     | ((hooks: Record<string, (...args: unknown[]) => unknown>) => void)
@@ -411,19 +444,47 @@ function installNodeModuleHooks() {
         context: Record<string, unknown>,
       ) => { url?: string } & Record<string, unknown>;
       const ctx = (context ?? {}) as Record<string, unknown>;
-      const result = next(specifier, ctx);
-      const resolvedUrl = result?.url;
-      if (typeof resolvedUrl === "string") {
-        maybeRecordDependency(resolvedUrl);
-      } else if (typeof specifier === "string") {
-        const resolveDir =
-          typeof ctx.parentURL === "string" &&
-          ctx.parentURL.startsWith("file://")
-            ? path.dirname(fileURLToPath(ctx.parentURL))
-            : undefined;
-        maybeRecordDependencyFromSpecifier(specifier, resolveDir);
+      try {
+        const result = next(specifier, ctx);
+        const resolvedUrl = result?.url;
+        if (typeof resolvedUrl === "string") {
+          maybeRecordDependency(resolvedUrl);
+        }
+        return result;
+      } catch (err) {
+        // When resolution fails for an extensionless relative/absolute specifier,
+        // try appending TypeScript extensions. This makes extensionless TS imports
+        // work without tsx hooks (e.g. `import { foo } from "./bar"` resolves to
+        // ./bar.ts).
+        if (
+          typeof specifier === "string" &&
+          isRelativeOrAbsoluteSpecifier(specifier) &&
+          !specifierHasExtension(specifier)
+        ) {
+          for (const ext of TS_RESOLVE_EXTENSIONS) {
+            try {
+              const result = next(specifier + ext, ctx);
+              const resolvedUrl = result?.url;
+              if (typeof resolvedUrl === "string") {
+                maybeRecordDependency(resolvedUrl);
+              }
+              return result;
+            } catch {
+              continue;
+            }
+          }
+        }
+        // Fall through: record dependency from the original specifier and rethrow.
+        if (typeof specifier === "string") {
+          const resolveDir =
+            typeof ctx.parentURL === "string" &&
+            ctx.parentURL.startsWith("file://")
+              ? path.dirname(fileURLToPath(ctx.parentURL))
+              : undefined;
+          maybeRecordDependencyFromSpecifier(specifier, resolveDir);
+        }
+        throw err;
       }
-      return result;
     },
   });
 }
@@ -749,14 +810,40 @@ function propagateInheritedBraintrustState(braintrust: BraintrustModule) {
   }
 }
 
+/**
+ * Detect if a file lives inside a `"type": "module"` package by walking
+ * up the directory tree looking for the nearest package.json.
+ */
+function isInsideEsmPackage(file: string): boolean {
+  let dir = path.dirname(file);
+  const root = path.parse(dir).root;
+  while (dir !== root) {
+    const pkgPath = path.join(dir, "package.json");
+    try {
+      const content = fsMutable.readFileSync(pkgPath, "utf8");
+      const pkg = JSON.parse(content);
+      return pkg.type === "module";
+    } catch {
+      // No package.json here, keep walking up.
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) {
+      break;
+    }
+    dir = parent;
+  }
+  return false;
+}
+
 async function loadFiles(files: string[]): Promise<unknown[]> {
   const modules: unknown[] = [];
   for (const file of files) {
     const fileUrl = pathToFileURL(file).href;
-    const preferRequire =
-      file.endsWith(".ts") || file.endsWith(".tsx") || file.endsWith(".cjs");
+    const isTsFile = file.endsWith(".ts") || file.endsWith(".tsx");
+    const isCjsFile = file.endsWith(".cjs") || file.endsWith(".cts");
 
-    if (preferRequire) {
+    // For .cjs/.cts files, always prefer require().
+    if (isCjsFile) {
       try {
         const require = createRequire(fileUrl);
         const mod = require(file);
@@ -775,6 +862,54 @@ async function loadFiles(files: string[]): Promise<unknown[]> {
       }
     }
 
+    // For TypeScript files in "type": "module" packages, prefer import() to
+    // avoid the require-of-ESM path which can corrupt the module graph on
+    // Node < 22.14 (see nodejs/node#54577). Only fall back to require() if
+    // import() fails with ERR_UNKNOWN_FILE_EXTENSION (no TS loader active).
+    if (isTsFile && isInsideEsmPackage(file)) {
+      try {
+        const mod = await import(fileUrl);
+        modules.push(mod);
+        continue;
+      } catch (esmErr) {
+        if (!shouldTryRequire(file, esmErr)) {
+          throw esmErr;
+        }
+        try {
+          const require = createRequire(fileUrl);
+          const mod = require(file);
+          modules.push(mod);
+          continue;
+        } catch (requireErr) {
+          throw new Error(
+            `Failed to load ${file} as ESM (${formatError(esmErr)}) or CJS (${formatError(requireErr)}).`,
+          );
+        }
+      }
+    }
+
+    // For TS files NOT in an ESM package, prefer require() (original behavior,
+    // works well with tsx CJS hooks).
+    if (isTsFile) {
+      try {
+        const require = createRequire(fileUrl);
+        const mod = require(file);
+        modules.push(mod);
+        continue;
+      } catch (requireErr) {
+        try {
+          const mod = await import(fileUrl);
+          modules.push(mod);
+          continue;
+        } catch (esmErr) {
+          throw new Error(
+            `Failed to load ${file} as CJS (${formatError(requireErr)}) or ESM (${formatError(esmErr)}).`,
+          );
+        }
+      }
+    }
+
+    // For .js/.mjs and other files, prefer import().
     try {
       const mod = await import(fileUrl);
       modules.push(mod);
@@ -1253,7 +1388,7 @@ async function createEvalRunner(config: RunnerConfig) {
 
 async function main() {
   const config = readRunnerConfig();
-  const files = process.argv.slice(2);
+  const files = readEvalFiles();
   if (files.length === 0) {
     console.error("No eval files provided.");
     process.exit(1);
@@ -1269,6 +1404,7 @@ async function main() {
   const braintrust = await loadBraintrust();
   propagateInheritedBraintrustState(braintrust);
   initRegistry();
+
   const modules = await loadFiles(normalized);
   const btEvalMains = collectBtEvalMains(modules);
 

src/eval.rs

@@ -284,6 +284,8 @@ async fn run_eval_files_once(
         EvalLanguage::JavaScript => build_js_command(runner_override, &js_runner, &files)?,
     };
 
+    let files_json = serde_json::to_string(&files).context("failed to serialize eval file list")?;
+    cmd.env("BT_EVAL_FILES", files_json);
     cmd.envs(build_env(base));
     if no_send_logs {
         cmd.env("BT_EVAL_NO_SEND_LOGS", "1");
@@ -691,52 +693,46 @@ fn build_js_command(
         let resolved_runner = resolve_js_runner_command(explicit, files);
         if is_deno_runner(explicit) || is_deno_runner_path(resolved_runner.as_ref()) {
             let runner_script = prepare_js_runner_in_cwd()?;
-            build_deno_js_command(resolved_runner.as_os_str(), &runner_script, files)
+            build_deno_js_command(resolved_runner.as_os_str(), &runner_script)
         } else {
             let runner_script = select_js_runner_entrypoint(runner, resolved_runner.as_ref())?;
             let mut command = Command::new(resolved_runner);
-            command.arg(runner_script).args(files);
+            command.arg(runner_script);
             command
         }
     } else if let Some(auto_runner) = find_js_runner_binary(files) {
         if is_deno_runner_path(&auto_runner) {
             let runner_script = prepare_js_runner_in_cwd()?;
-            build_deno_js_command(auto_runner.as_os_str(), &runner_script, files)
+            build_deno_js_command(auto_runner.as_os_str(), &runner_script)
         } else {
             let runner_script = select_js_runner_entrypoint(runner, auto_runner.as_ref())?;
             let mut command = Command::new(auto_runner);
-            command.arg(runner_script).args(files);
+            command.arg(runner_script);
             command
         }
     } else {
         let mut command = Command::new("npx");
-        command.arg("--yes").arg("tsx").arg(runner).args(files);
+        command.arg("--yes").arg("tsx").arg(runner);
         command
     };
 
     Ok(command)
 }
 
-fn build_deno_js_command(
-    deno_runner: impl AsRef<OsStr>,
-    runner: &Path,
-    files: &[String],
-) -> Command {
+fn build_deno_js_command(deno_runner: impl AsRef<OsStr>, runner: &Path) -> Command {
     let mut command = Command::new(deno_runner);
-    command.args(deno_js_command_args(runner, files));
+    command.args(deno_js_command_args(runner));
     command
 }
 
-fn deno_js_command_args(runner: &Path, files: &[String]) -> Vec<OsString> {
-    let mut args = vec![
+fn deno_js_command_args(runner: &Path) -> Vec<OsString> {
+    vec![
         OsString::from("run"),
         OsString::from("-A"),
         OsString::from("--node-modules-dir=auto"),
         OsString::from("--unstable-detect-cjs"),
         runner.as_os_str().to_os_string(),
-    ];
-    args.extend(files.iter().map(OsString::from));
-    args
+    ]
 }
 
 fn build_python_command(
@@ -1865,8 +1861,7 @@ mod tests {
     #[test]
     fn build_deno_js_command_includes_detect_cjs_flag() {
         let runner = PathBuf::from("/tmp/eval-runner.ts");
-        let files = vec!["tests/basic.eval.ts".to_string()];
-        let args: Vec<String> = deno_js_command_args(&runner, &files)
+        let args: Vec<String> = deno_js_command_args(&runner)
             .into_iter()
             .map(|arg| arg.to_string_lossy().to_string())
             .collect();
@@ -1878,7 +1873,6 @@ mod tests {
                 "--node-modules-dir=auto",
                 "--unstable-detect-cjs",
                 "/tmp/eval-runner.ts",
-                "tests/basic.eval.ts",
             ]
         );
     }

tests/eval_fixtures.rs

@@ -18,6 +18,7 @@ struct FixtureConfig {
     env: Option<BTreeMap<String, String>>,
     args: Option<Vec<String>>,
     expect_success: Option<bool>,
+    min_node_version: Option<u32>,
 }
 
 fn test_lock() -> MutexGuard<'static, ()> {
@@ -88,6 +89,12 @@ fn eval_fixtures() {
                 continue;
             }
         }
+        if let Some(min_ver) = config.min_node_version {
+            if node_major_version().is_none_or(|v| v < min_ver) {
+                eprintln!("Skipping {fixture_name} (requires node >= {min_ver}).");
+                continue;
+            }
+        }
         match runtime {
             "node" => ensure_dependencies(&dir),
             "bun" => ensure_dependencies(&dir),
@@ -648,6 +655,20 @@ fn build_bt_binary(root: &Path) {
     }
 }
 
+fn node_major_version() -> Option<u32> {
+    let output = Command::new("node")
+        .args([
+            "-e",
+            "process.stdout.write(process.versions.node.split('.')[0])",
+        ])
+        .output()
+        .ok()?;
+    if !output.status.success() {
+        return None;
+    }
+    String::from_utf8_lossy(&output.stdout).trim().parse().ok()
+}
+
 fn command_exists(command: &str) -> bool {
     let paths = match std::env::var_os("PATH") {
         Some(paths) => paths,

tests/evals/js/eval-ts-esm-monorepo/fixture.json

@@ -0,0 +1,5 @@
+{
+  "runner": "./run-node-ts",
+  "files": ["tests/basic.eval.ts"],
+  "min_node_version": 22
+}

tests/evals/js/eval-ts-esm-monorepo/package.json

@@ -0,0 +1,12 @@
+{
+  "name": "bt-eval-ts-esm-monorepo",
+  "private": true,
+  "type": "module",
+  "workspaces": [
+    "packages/*"
+  ],
+  "dependencies": {
+    "@repo/lib": "workspace:*",
+    "braintrust": "^2.2.0"
+  }
+}

tests/evals/js/eval-ts-esm-monorepo/packages/lib/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "@repo/lib",
+  "version": "0.0.1",
+  "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  }
+}

tests/evals/js/eval-ts-esm-monorepo/packages/lib/src/helper.ts

@@ -0,0 +1,8 @@
+// Extensionless import — this is the pattern that breaks without the fix.
+// Node's ESM resolver requires explicit extensions, so `./utils` (without .ts)
+// fails unless a resolve hook adds the extension.
+import { greet } from "./utils";
+
+export function greetAll(names: string[]): string[] {
+  return names.map(greet);
+}

tests/evals/js/eval-ts-esm-monorepo/packages/lib/src/index.ts

@@ -0,0 +1,2 @@
+export { greet } from "./utils.ts";
+export { greetAll } from "./helper.ts";

tests/evals/js/eval-ts-esm-monorepo/packages/lib/src/utils.ts

@@ -0,0 +1,3 @@
+export function greet(name: string): string {
+  return `Hello ${name}`;
+}