Add Agent Eval assertions on files

scripts/agent-evals/package.json

@@ -11,8 +11,11 @@
   "author": "",
   "license": "ISC",
   "devDependencies": {
+    "@types/chai": "^4.3.0",
+    "@types/chai-fs": "^2.0.5",
     "@types/mocha": "^9.0.0",
-    "@types/node": "^24.9.0"
+    "@types/node": "^24.9.0",
+    "chai-fs": "^2.0.0"
   },
   "dependencies": {
     "mocha": "^11.7.1",

scripts/agent-evals/src/runner/agent-test-runner.ts

@@ -19,4 +19,10 @@ export interface AgentTestRunner {
    * an event is not found
    */
   expectToolCalls(tools: ToolDef[]): Promise<void>;
+
+  /**
+   * Returns an assertion object for the path inside the run directory that can
+   * be asserted on via chai-fs's API: https://www.chaijs.com/plugins/chai-fs/
+   */
+  expectFs(filePath: string): Chai.Assertion;
 }

scripts/agent-evals/src/runner/expect-files.ts

@@ -0,0 +1,10 @@
+import * as chai from "chai";
+import chaiFs from "chai-fs";
+
+// Prevent file paths in assertions from being cut off
+chai.config.truncateThreshold = 0;
+chai.use(chaiFs);
+
+export function expectFile(path: string): Chai.Assertion {
+  return chai.expect(path);
+}