Add black-box e2e tests driving the nix-built binary

README.md

@@ -209,7 +209,8 @@ instead of `github:juspay/odu`).
 ```sh
 just install     # pnpm install + hydrate @kolu/* from the npins kolu pin
 just typecheck
-just test        # the loopback falsifiability suite
+just test        # unit tests — the loopback falsifiability suite
+just e2e         # black-box e2e — nix-build odu, drive it against a fixture repo
 just run -- run --no-strict fmt   # one recipe, locally, against the live tree
 ```
 

ci/mod.just

@@ -9,7 +9,7 @@ nix_shell := if env('IN_NIX_SHELL', '') != '' { '' } else { 'nix develop --accep
 
 [parallel]
 [metadata("ci")]
-default: typecheck unit fmt nix
+default: typecheck unit fmt nix e2e
 
 install:
     {{ nix_shell }} pnpm install --frozen-lockfile
@@ -24,6 +24,12 @@ typecheck: install
 unit: install
     {{ nix_shell }} pnpm test:unit
 
+# Black-box e2e: builds the odu binary with nix and drives it against a
+# throwaway fixture repo on a localhost lane (tests/e2e/README.md). Needs the
+# dev shell on PATH (nix, git, just) plus the vitest/tsx deps from `install`.
+e2e: install
+    {{ nix_shell }} pnpm test:e2e
+
 fmt:
     {{ nix_shell }} nixpkgs-fmt --check *.nix nix/*.nix nix/packages/*.nix
 

justfile

@@ -24,6 +24,11 @@ typecheck: install
 test: install
     {{ nix_shell }} pnpm test:unit
 
+# Black-box e2e: build the odu binary with nix and drive it against a
+# throwaway fixture repo on a localhost lane (tests/e2e/README.md).
+e2e: install
+    {{ nix_shell }} pnpm test:e2e
+
 # Run odu from source: `just run -- run --no-strict biome`
 run *args: install
     {{ nix_shell }} pnpm start {{ args }}

package.json

@@ -8,7 +8,8 @@
     "start": "tsx src/cli/main.ts",
     "start:runner": "tsx src/runner/main.ts --stdio",
     "typecheck": "tsc --noEmit",
-    "test:unit": "vitest run"
+    "test:unit": "vitest run",
+    "test:e2e": "vitest run --config vitest.e2e.config.ts"
   },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",

tests/e2e/README.md

@@ -0,0 +1,72 @@
+# E2E tests
+
+Black-box end-to-end tests: build the real `odu` binary with Nix, run it
+against a throwaway fixture repo on a **localhost lane**, and assert on its
+`--progress json` stream and process exit code.
+
+These cover the seams the in-process loopback suite (`src/odu.test.ts`) stubs:
+just-DAG ingest → scheduling → local lane spawn (`odu-runner`) → NDJSON
+projection → exit code.
+
+## Layout
+
+```
+tests/e2e/
+├── harness.ts            # nix build, fixture materialization, run + parse
+├── run.e2e.test.ts       # the assertions (Vitest)
+├── fixtures/
+│   ├── _flake.nix.in     # shared flake template (re-exports odu-runner)
+│   ├── pass/justfile     # a DAG that goes green
+│   └── fail/justfile     # a DAG whose node fails (exit 1)
+└── README.md
+```
+
+Run locally:
+
+```sh
+pnpm test:e2e            # vitest run --config vitest.e2e.config.ts
+```
+
+In CI it's the `e2e` step in `ci/mod.just`.
+
+## How a fixture works
+
+A local `odu run` resolves its lane runner by evaluating
+`<repo>#packages.<system>.odu-runner.drvPath` (`src/coordinator/run.ts`) — this
+happens even for a localhost lane (the realise is a local no-op copy, but the
+drvPath lookup still runs). So **any repo that runs odu locally must expose
+`odu-runner` in its flake.** Each fixture is therefore a flake: the harness
+rewrites `__ODU_FLAKE__` in `_flake.nix.in` to a `path:` input pointing at the
+checkout under test and re-exports its packages, so the fixture runs the exact
+`odu-runner` the harness just built (a Nix cache hit).
+
+The leaf recipes are pure shell — the fixture's own "CI" is trivial on purpose,
+so the test exercises *odu's* machinery, not a real toolchain.
+
+## Deliberate tradeoffs
+
+- **The harness builds the binary itself** (`nix build .#odu .#odu-runner`)
+  rather than assuming a pre-built `./result`. This makes the suite
+  self-contained and order-independent, at the cost of a cold-cache build on
+  the first run (hence the 10-minute `beforeAll` timeout). Subsequent runs are
+  cache hits.
+- **Assertions read `--progress json` (NDJSON), not the TTY dashboard.** The
+  dashboard is a separate rendering path; asserting on the clean, parseable
+  stream keeps tests robust. The dashboard is currently uncovered by e2e.
+- **Local-only.** No remote/ssh lanes are exercised; fixtures run on a
+  localhost lane against the live working tree (`--no-strict`).
+- **Black-box.** `harness.ts` imports nothing from `src/` — the contract under
+  test is the binary's observable behavior, so internal refactors don't ripple
+  into these tests.
+
+## Follow-ups
+
+- Cover the **TTY dashboard** / `attach` live view (PTY-driven).
+- Exercise a **real transport**: an ssh-to-localhost lane to cover the
+  `nix copy` → remote realise → spawn-over-ssh path that localhost
+  short-circuits.
+- Drive the **MCP agent face** (`odu mcp`) end-to-end as a subprocess.
+- Cover `status -o json` and `logs -f` against a live run (they need the
+  `.ci/odu.sock` socket, so the harness would run `odu run` in the background
+  and dial it concurrently).
+- **Multi-platform fanout** once remote lanes are in scope.

tests/e2e/fixtures/_flake.nix.in

@@ -0,0 +1,13 @@
+{
+  # E2E fixture flake. A local `odu run` resolves the lane runner by
+  # evaluating `<repo>#packages.<system>.odu-runner.drvPath` (see
+  # src/coordinator/run.ts) — even for a localhost lane (the realise is a
+  # local no-op copy, but the drvPath lookup still happens). So any repo
+  # that runs odu locally must expose `odu-runner` in its flake.
+  #
+  # The harness rewrites the odu input url below to a `path:` input pointing at
+  # the odu checkout under test and we re-export its packages verbatim, so the
+  # fixture runs the exact `odu-runner` the harness already built (a cache hit).
+  inputs.odu.url = "__ODU_FLAKE__";
+  outputs = { odu, ... }: { packages = odu.packages; };
+}

tests/e2e/fixtures/fail/justfile

@@ -0,0 +1,13 @@
+# A failing fixture DAG. `boom` exits non-zero, so its node goes red and
+# `odu run` exits 1 — the path that proves odu propagates a node failure all
+# the way out to the process exit code.
+
+[parallel]
+[metadata("ci")]
+default: ok boom
+
+ok:
+    echo "ok ran"
+
+boom: ok
+    exit 1

tests/e2e/fixtures/pass/justfile

@@ -0,0 +1,14 @@
+# A passing fixture DAG. odu discovers the [metadata("ci")] recipe, expands
+# its reachable subgraph, and runs each reachable recipe as
+# `just --no-deps <name>`. Every leaf here is pure shell that exits 0, so the
+# whole run goes green and `odu run` exits 0.
+
+[parallel]
+[metadata("ci")]
+default: alpha beta
+
+alpha:
+    echo "alpha ran"
+
+beta: alpha
+    echo "beta ran"

tests/e2e/harness.ts

@@ -0,0 +1,159 @@
+/**
+ * Black-box e2e harness: drive the real, nix-built `odu` binary against a
+ * throwaway fixture repo and read back its `--progress json` stream.
+ *
+ * No imports from `src/` on purpose — the contract under test is the binary's
+ * observable behavior (NDJSON shape + exit code), not its internals. See
+ * tests/e2e/README.md for the design tradeoffs this harness commits to.
+ */
+
+import { execFileSync, spawnSync } from "node:child_process";
+import {
+  cpSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const here = dirname(fileURLToPath(import.meta.url));
+/** Large maxBuffer for nix output / NDJSON streams (256 MiB). */
+export const BIG = 256 * 1024 * 1024;
+
+/** The odu checkout under test — the worktree this test file lives in. */
+export const repoRoot = execFileSync(
+  "git",
+  ["rev-parse", "--show-toplevel"],
+  { cwd: here, encoding: "utf-8" },
+).trim();
+
+/** `path:` flake ref the fixture re-exports `odu-runner` from. */
+export const oduFlakeRef = `path:${repoRoot}`;
+
+/** Substituted `flake.nix` content, computed once and reused across fixtures. */
+const flakeNix: string = readFileSync(
+  join(here, "fixtures", "_flake.nix.in"),
+  "utf-8",
+).replaceAll("__ODU_FLAKE__", oduFlakeRef);
+
+/**
+ * One line of `odu run --progress json` output. Deliberately mirrors — and is
+ * NOT imported from — `ProgressEvent` in src/coordinator/display.ts: this is
+ * the wire schema the test deserializes, so the assertions verify the binary's
+ * real output (black-box). Sharing the type would make the test white-box and
+ * hide exactly the wire-format regressions this suite exists to catch.
+ */
+export interface ProgressEvent {
+  node: string; // fanId, e.g. "alpha@x86_64-linux"
+  recipe: string; // just namepath, e.g. "alpha"
+  platform: string; // e.g. "x86_64-linux"
+  status: "running" | "success" | "failed" | "skipped" | "errored";
+  exit_code?: number;
+  log: string;
+}
+
+export interface RunResult {
+  status: number | null;
+  events: ProgressEvent[];
+  stdout: string;
+  stderr: string;
+}
+
+/**
+ * Build `odu` (and warm `odu-runner`) from the worktree and return the path to
+ * the `odu` executable. Prebuilding the runner means the fixture's in-flight
+ * `nix eval …odu-runner.drvPath` resolves to an already-realised store path.
+ */
+export function buildOduBinary(): string {
+  const build = (attr: string): string =>
+    execFileSync(
+      "nix",
+      [
+        "build",
+        attr,
+        "--no-link",
+        "--print-out-paths",
+        "--accept-flake-config",
+      ],
+      { cwd: repoRoot, encoding: "utf-8", maxBuffer: BIG },
+    ).trim();
+
+  const oduOut = build(".#odu");
+  build(".#odu-runner"); // warm the store path the fixture will realise
+  return join(oduOut, "bin", "odu");
+}
+
+/**
+ * Materialize a fixture into a fresh temp git repo: the named fixture's
+ * `justfile` plus a `flake.nix` re-exporting `odu-runner` from the checkout
+ * under test. Flakes only see git-tracked files, so we commit before returning.
+ */
+export function makeFixture(name: string): string {
+  const dir = mkdtempSync(join(tmpdir(), `odu-e2e-${name}-`));
+  cpSync(join(here, "fixtures", name), dir, { recursive: true });
+
+  writeFileSync(join(dir, "flake.nix"), flakeNix);
+
+  const git = (...args: string[]): void => {
+    execFileSync("git", args, { cwd: dir, encoding: "utf-8" });
+  };
+  git("init", "-q");
+  git("add", "-A");
+  git(
+    "-c",
+    "user.email=e2e@odu.test",
+    "-c",
+    "user.name=odu e2e",
+    "commit",
+    "-q",
+    "-m",
+    "fixture",
+  );
+  return dir;
+}
+
+/** Run `odu run --no-strict --progress json` in `dir` and parse the stream. */
+export function oduRun(
+  oduBin: string,
+  dir: string,
+  selectors: string[] = [],
+): RunResult {
+  const res = spawnSync(
+    oduBin,
+    ["run", "--no-strict", "--progress", "json", ...selectors],
+    { cwd: dir, encoding: "utf-8", maxBuffer: BIG },
+  );
+  const events: ProgressEvent[] = [];
+  for (const line of res.stdout.split("\n")) {
+    if (line.trim() === "") continue;
+    try {
+      events.push(JSON.parse(line) as ProgressEvent);
+    } catch (err) {
+      // non-JSON noise on stdout is a contract violation — log it so
+      // regressions surface here rather than as a mysterious missing-event
+      // assertion failure downstream.
+      process.stderr.write(`e2e: unparseable NDJSON line: ${line}\n${String(err)}\n`);
+    }
+  }
+  return { status: res.status, events, stdout: res.stdout, stderr: res.stderr };
+}
+
+/** The terminal (last-seen) status for each recipe across the event stream. */
+export function terminalStatuses(
+  events: ProgressEvent[],
+): Map<string, ProgressEvent> {
+  return new Map(events.map((e) => [e.recipe, e]));
+}
+
+/** Best-effort temp-dir cleanup; never throws, but a failure is logged so a
+ *  leaked fixture dir is visible in CI rather than silently accumulating. */
+export function cleanup(dir: string): void {
+  try {
+    rmSync(dir, { recursive: true, force: true });
+  } catch (err) {
+    process.stderr.write(`e2e: failed to remove fixture ${dir}: ${String(err)}\n`);
+  }
+}