test(test262): #5346 — run Node oracle as a global script (vm.runInThisContext)

scripts/test262_subset.py

@@ -92,6 +92,12 @@
 REPO_ROOT = SCRIPT_DIR.parent
 TEST262_DIR = REPO_ROOT / "test-compat" / "test262"
 PREAMBLE = TEST262_DIR / "preamble.js"
+# Node host shim: runs an assembled case as a *global script* (via
+# vm.runInThisContext) rather than a CommonJS module, matching a real Test262
+# host and Perry. Without it, module-scoped harness intrinsics are invisible to
+# global-scope indirect eval, so Node spuriously rejects Annex B eval cases
+# (#5346).
+HOST_RUNNER = TEST262_DIR / "host-run.cjs"
 # Feature tags Perry implements but the Node oracle cannot run (e.g. Temporal).
 # Cases tagged with one of these are judged in self-validating mode (#4792):
 # Perry-only, scored on whether the case's own `assert.*` self-checks throw.
@@ -473,7 +479,8 @@ def judge_one(case):
                           else "ran clean; expected a thrown error (negative)")
                 return (rel, cat, "runtime-fail", reason, False, True)
             # 1) Node is the oracle (negative cases legitimately exit != 0).
-            n_exit, n_out = run(["node", str(staged)], base_env, args.timeout)
+            n_exit, n_out = run(["node", str(HOST_RUNNER), str(staged)],
+                                base_env, args.timeout)
             node_clean = n_exit == 0
             # 2) Perry compile (permissive — unimplemented surfaces as a gap).
             out_bin = workdir / "case.out"

test-compat/test262/README.md

@@ -78,6 +78,17 @@ no harness. Because both runtimes load the *same* assembled script, the
 differential compares the two runtimes' **builtins**, never their
 harnesses.
 
+The Node oracle runs the assembled script through `host-run.cjs`
+(`vm.runInThisContext`) rather than `node case.js` directly. `node file.js`
+evaluates a file as a **CommonJS module**, scoping its top-level
+`var`/`function` declarations to the module wrapper instead of the global
+object — which diverges from both a conforming Test262 host and Perry, and
+breaks any case whose harness intrinsics must be reachable from global
+scope (notably the Annex B `eval-code/indirect` family, where an indirect
+`(0,eval)(...)` runs in global scope and otherwise can't see a
+module-scoped `assert`). `runInThisContext` restores true global-script
+semantics so the oracle agrees with a conforming host (#5346).
+
 Raw CommonJS/JS runs under Perry because Perry feeds user `.js` through
 the native AOT pipeline (the same path `compilePackages` uses; see #668).
 
@@ -116,6 +127,8 @@ the negative-rejection signal stay legible.
 - `pinned-sha.txt` — the Test262 SHA the corpus is pulled from.
 - `preamble.js` — host shims (`print`, `$DONOTEVALUATE`) prepended to
   every non-`raw` assembled case under both runtimes.
+- `host-run.cjs` — Node oracle entry point; runs an assembled case as a
+  global script via `vm.runInThisContext` (see the harness note above, #5346).
 - `report.json` — written by the runner (a generated artifact; not
   committed).
 

test-compat/test262/host-run.cjs

@@ -0,0 +1,29 @@
+// Node host runner for the Test262 subset radar (#799, #5346).
+//
+// `node file.js` executes the file as a CommonJS module: its top-level
+// `var`/`function` declarations live in the module wrapper, NOT on the global
+// object. That diverges from a real Test262 host (and from Perry), which run
+// each assembled case as a *global script*. The difference is invisible for
+// most cases but breaks any test whose harness intrinsics must be reachable
+// from global scope — notably the Annex B `eval-code/indirect` family, where
+// an indirect `(0,eval)(...)` evaluates in the global scope and so cannot see
+// a module-scoped `assert` (it throws `ReferenceError: assert is not defined`,
+// making Node spuriously "reject" a case Perry runs clean).
+//
+// Running the assembled program through `vm.runInThisContext` restores true
+// script semantics: top-level declarations become globals, top-level `this` is
+// `globalThis`, and a syntax error still throws at compile time (so negative
+// parse cases keep exiting non-zero). This makes the Node oracle agree with a
+// conforming host instead of with the CommonJS wrapper.
+//
+// The `.cjs` extension is deliberate: the repo's package.json sets
+// `"type": "module"`, so a `.js` shim would load as ESM and `require` would be
+// undefined. `.cjs` forces CommonJS for the shim itself; the case it runs is
+// still evaluated as a global script via vm.runInThisContext.
+"use strict";
+const vm = require("vm");
+const fs = require("fs");
+
+const file = process.argv[2];
+const code = fs.readFileSync(file, "utf8");
+vm.runInThisContext(code, { filename: file });