Bring CI under two minutes

.github/workflows/ci.yml

@@ -144,15 +144,18 @@ jobs:
 
   test:
     name: Test
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: blacksmith-16vcpu-ubuntu-2404
     timeout-minutes: 15
-    # Tuned for Blacksmith's 4 vCPU runners.
+    # Run eight independent packages at once and cap each Vitest process at two
+    # workers. This uses all 16 vCPUs without every nested runner seeing the
+    # whole machine and oversubscribing it.
     env:
       TURBO_API: ${{ vars.TURBO_API }}
       TURBO_TEAM: ${{ vars.TURBO_TEAM }}
       TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
       TURBO_REMOTE_CACHE_SIGNATURE_KEY: ${{ secrets.TURBO_REMOTE_CACHE_SIGNATURE_KEY }}
-      TURBO_TEST_CONCURRENCY: 4
+      TURBO_TEST_CONCURRENCY: 8
+      VITEST_MAX_WORKERS: 2
     steps:
       - uses: actions/checkout@v4
 
@@ -184,6 +187,8 @@ jobs:
       # no signal. This is just a few file reads plus greps, so ~zero cost.
       - run: bun run check:patches
 
+      - run: bun run --cwd e2e test:ci-shard
+
       - run: bun run test
 
   e2e:
@@ -192,25 +197,41 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # PGlite is deliberately single-connection, and under a sustained
-          # multi-minute shard it can stop accepting postgres sockets. Keep
-          # every hermetic dev stack short: eight serial shards remove that
-          # lifetime-dependent failure and put cloud below the selfhost lane.
-          - { target: cloud, shard: 1/8, shard-name: 1of8 }
-          - { target: cloud, shard: 2/8, shard-name: 2of8 }
-          - { target: cloud, shard: 3/8, shard-name: 3of8 }
-          - { target: cloud, shard: 4/8, shard-name: 4of8 }
-          - { target: cloud, shard: 5/8, shard-name: 5of8 }
-          - { target: cloud, shard: 6/8, shard-name: 6of8 }
-          - { target: cloud, shard: 7/8, shard-name: 7of8 }
-          - { target: cloud, shard: 8/8, shard-name: 8of8 }
-          # Selfhost shards the same way: each shard is its own runner booting
-          # its own fresh instance (own port block + data dir), so the
-          # project's shared-bootstrap-admin assumption stays intact per shard
-          # and `fileParallelism: false` still serializes within a shard.
-          - { target: selfhost, shard: 1/3, shard-name: 1of3 }
-          - { target: selfhost, shard: 2/3, shard-name: 2of3 }
-          - { target: selfhost, shard: 3/3, shard-name: 3of3 }
+          # The planner assigns every file exactly once using recorded slow-file
+          # durations plus a conservative weight for new tests. The cloud DB's
+          # connection teardown and concurrent socket protocol have dedicated
+          # regression tests; these shards balance wall clock, not hide retries.
+          - { target: cloud, shard-index: 1, shard-name: 1of16 }
+          - { target: cloud, shard-index: 2, shard-name: 2of16 }
+          - { target: cloud, shard-index: 3, shard-name: 3of16 }
+          - { target: cloud, shard-index: 4, shard-name: 4of16 }
+          - { target: cloud, shard-index: 5, shard-name: 5of16 }
+          - { target: cloud, shard-index: 6, shard-name: 6of16 }
+          - { target: cloud, shard-index: 7, shard-name: 7of16 }
+          - { target: cloud, shard-index: 8, shard-name: 8of16 }
+          - { target: cloud, shard-index: 9, shard-name: 9of16 }
+          - { target: cloud, shard-index: 10, shard-name: 10of16 }
+          - { target: cloud, shard-index: 11, shard-name: 11of16 }
+          - { target: cloud, shard-index: 12, shard-name: 12of16 }
+          - { target: cloud, shard-index: 13, shard-name: 13of16 }
+          - { target: cloud, shard-index: 14, shard-name: 14of16 }
+          - { target: cloud, shard-index: 15, shard-name: 15of16 }
+          - { target: cloud, shard-index: 16, shard-name: 16of16 }
+          - { target: selfhost, shard-index: 1, shard-name: 1of10 }
+          - { target: selfhost, shard-index: 2, shard-name: 2of10 }
+          - { target: selfhost, shard-index: 3, shard-name: 3of10 }
+          - { target: selfhost, shard-index: 4, shard-name: 4of10 }
+          - { target: selfhost, shard-index: 5, shard-name: 5of10 }
+          - { target: selfhost, shard-index: 6, shard-name: 6of10 }
+          - { target: selfhost, shard-index: 7, shard-name: 7of10 }
+          - { target: selfhost, shard-index: 8, shard-name: 8of10 }
+          - { target: selfhost, shard-index: 9, shard-name: 9of10 }
+          - { target: selfhost, shard-index: 10, shard-name: 10of10 }
+          # Local files own their server, browser and data directory. Separate
+          # runners preserve that isolation while removing its 69-second serial
+          # lane from the two-minute critical path.
+          - { target: local, shard-index: 1, shard-name: 1of2 }
+          - { target: local, shard-index: 2, shard-name: 2of2 }
     runs-on: blacksmith-4vcpu-ubuntu-2404
     timeout-minutes: 30
     steps:
@@ -246,26 +267,29 @@ jobs:
 
       # Install from e2e so bunx resolves ITS pinned playwright (the version
       # the tests run against) rather than floating to the latest.
+      # Blacksmith's Ubuntu image carries the official GitHub runner system
+      # dependencies. Restore the pinned browser binaries without apt-updating
+      # every matrix machine.
       - name: Install Playwright Chromium
-        run: bunx playwright install --with-deps chromium chromium-headless-shell
+        run: bunx playwright install chromium chromium-headless-shell
         working-directory: e2e
 
-      # The globalsetup boots the target's own dev server (ports are claimed
-      # per checkout, so this is hermetic) and tears it down after the run.
+      # Each target either boots its own shared dev server or lets each file own
+      # its server. Ports and data paths are hermetic in both cases.
       # Do not retry scenarios: retries hide flakes and multiply slow timeout
       # failures. The fixtures and process lifecycle are deterministic enough
       # that the first result is the result.
-      - name: Run cloud scenarios
+      - name: Run cloud shard
         if: matrix.target == 'cloud'
         env:
           MCP_SESSION_TIMEOUT_MS: "3000"
           MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000"
-        run: bunx vitest run --project cloud ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
+        run: bun scripts/run-ci-shard.ts cloud ${{ matrix['shard-index'] }}
         working-directory: e2e
 
-      - name: Run selfhost scenarios
-        if: matrix.target == 'selfhost'
-        run: bunx vitest run --project selfhost ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
+      - name: Run scenarios
+        if: matrix.target != 'cloud'
+        run: bun scripts/run-ci-shard.ts ${{ matrix.target }} ${{ matrix['shard-index'] }}
         working-directory: e2e
 
       # Failed runs keep their trace.zip / session.mp4 / step screenshots in
@@ -278,56 +302,6 @@ jobs:
           path: e2e/runs/
           retention-days: 7
 
-  e2e-local:
-    name: E2E (local)
-    runs-on: blacksmith-4vcpu-ubuntu-2404
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v4
-
-      - uses: oven-sh/setup-bun@v2
-        with:
-          bun-version: 1.3.11
-
-      - name: Cache Bun package cache
-        uses: actions/cache@v4
-        with:
-          path: ~/.bun/install/cache
-          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
-          restore-keys: |
-            ${{ runner.os }}-bun-1.3.11-
-
-      # The local scenarios boot a real `executor web` (which spawns a Node
-      # sidecar) and some drive a browser, so pin Node 24 and install Chromium.
-      - uses: actions/setup-node@v4
-        with:
-          node-version: 24
-
-      - run: bun install --frozen-lockfile
-
-      - name: Cache Playwright browsers
-        uses: actions/cache@v4
-        with:
-          path: ~/.cache/ms-playwright
-          key: ${{ runner.os }}-playwright-1.60.0
-          restore-keys: |
-            ${{ runner.os }}-playwright-
-
-      # `chromium` and the new `chromium-headless-shell` ship as separate
-      # downloads; the browser-driven scenarios launch the headless shell.
-      # Install from e2e so bunx resolves ITS pinned playwright (the version the
-      # tests run against) rather than floating to the latest, which would fetch
-      # a browser build the test runtime does not look for.
-      - name: Install Playwright Chromium
-        run: bunx playwright install --with-deps chromium chromium-headless-shell
-        working-directory: e2e
-
-      # Each scenario owns its server, browser, data directory, and descendants;
-      # run the complete hermetic suite on PRs without scenario retries.
-      - name: Run local scenarios
-        run: bunx vitest run --project local
-        working-directory: e2e
-
   desktop-smoke:
     name: Desktop smoke build
     needs: changes

apps/cloud/scripts/test-globalsetup.ts

@@ -13,12 +13,26 @@ import { fileURLToPath } from "node:url";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-const PORT = 5434;
+const parsePort = (input: string | undefined): number => {
+  if (input === undefined) return 5434;
+  if (!/^\d+$/.test(input)) throw new Error("CLOUD_TEST_DB_PORT must be an integer");
+  const port = Number(input);
+  if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) {
+    throw new Error("CLOUD_TEST_DB_PORT must be between 1 and 65535");
+  }
+  return port;
+};
+
+const PORT = parsePort(process.env.CLOUD_TEST_DB_PORT);
 const MIGRATIONS_FOLDER = resolve(__dirname, "../drizzle");
 
 let db: PGlite | undefined;
 let server: PGLiteSocketServer | undefined;
 
+/**
+ * Starts the cloud unit-test database and returns teardown that releases every
+ * resource without allowing PGlite shutdown to erase Vitest's failure status.
+ */
 export default async function setup() {
   db = await PGlite.create();
   await migrate(drizzle(db), { migrationsFolder: MIGRATIONS_FOLDER });
@@ -30,7 +44,12 @@ export default async function setup() {
   console.log(`[test-db] PGlite socket server listening on 127.0.0.1:${PORT}`);
 
   return async () => {
+    // PGlite sets an internal 99 sentinel on startup and replaces it with 0 on
+    // close. Preserve Vitest's failure status across that close so
+    // teardown can turn neither a red test green nor a green test red.
+    const testsFailed = process.exitCode === 1;
     await server?.stop();
     await db?.close();
+    if (testsFailed) process.exitCode = 1;
   };
 }

apps/cloud/src/account/org-api-key-revoke.node.test.ts

@@ -6,6 +6,7 @@ import { AccountError, AccountForbidden } from "@executor-js/api";
 
 import { ApiKeyService, OrgApiKeyNotFound } from "../auth/api-keys";
 import { UserStoreService } from "../auth/context";
+import { ORG_SELECTOR_HEADER } from "../auth/organization";
 import { WorkOSClient, type WorkOSClientService } from "../auth/workos";
 import { AutumnService } from "../extensions/billing/service";
 import { AccountCaller, workosAccountProvider } from "./workos-account-service";
@@ -36,6 +37,7 @@ const MEMBER = "user_member";
 const ORG_KEY = "key_org_1";
 const USER_KEY = "key_user_1";
 const createdAt = new Date("2026-01-01T00:00:00.000Z");
+const orgHeaders = { [ORG_SELECTOR_HEADER]: ORG };
 
 const session = (accountId: string) => ({
   accountId,
@@ -157,7 +159,7 @@ describe("revokeOrgApiKey · provider boundary", () => {
       const { provider, revoked } = providerWith(ADMIN);
       const account = yield* provider;
 
-      const result = yield* account.revokeOrgApiKey({}, ORG_KEY);
+      const result = yield* account.revokeOrgApiKey(orgHeaders, ORG_KEY);
 
       expect(result).toEqual({ success: true });
       expect(revoked, "the revoke reached the key service").toEqual([ORG_KEY]);
@@ -169,7 +171,7 @@ describe("revokeOrgApiKey · provider boundary", () => {
       const { provider, revoked } = providerWith(MEMBER);
       const account = yield* provider;
 
-      const error = yield* Effect.flip(account.revokeOrgApiKey({}, ORG_KEY));
+      const error = yield* Effect.flip(account.revokeOrgApiKey(orgHeaders, ORG_KEY));
 
       expect(error, "same admin gate as the mint").toBeInstanceOf(AccountForbidden);
       expect(revoked, "the gate runs BEFORE the key service is touched").toEqual([]);
@@ -183,7 +185,7 @@ describe("revokeOrgApiKey · provider boundary", () => {
       const { provider, revoked } = providerWith(ADMIN);
       const account = yield* provider;
 
-      const error = yield* Effect.flip(account.revokeOrgApiKey({}, USER_KEY));
+      const error = yield* Effect.flip(account.revokeOrgApiKey(orgHeaders, USER_KEY));
 
       expect(error).toBeInstanceOf(AccountError);
       expect(revoked).toEqual([]);

apps/cloud/src/test-globalsetup-exit.node.test.ts

@@ -0,0 +1,41 @@
+import { spawnSync } from "node:child_process";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+import { describe, expect, it } from "@effect/vitest";
+
+const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const vitestBin = resolve(appRoot, "../../node_modules/vitest/vitest.mjs");
+const fixtureConfig = resolve(appRoot, "test-fixtures/test-globalsetup-exit/vitest.config.ts");
+
+const runFixture = (port: number, shouldPass: boolean) =>
+  spawnSync(process.execPath, [vitestBin, "run", "--config", fixtureConfig], {
+    cwd: appRoot,
+    encoding: "utf8",
+    timeout: 60_000,
+    env: {
+      ...process.env,
+      CLOUD_TEST_DB_PORT: String(port),
+      TEST_GLOBALSETUP_SHOULD_PASS: String(shouldPass),
+    },
+  });
+
+const diagnostic = (result: ReturnType<typeof runFixture>): string =>
+  [result.stdout, result.stderr].filter(Boolean).join("\n");
+
+describe("cloud test global setup", () => {
+  it("does not let PGlite teardown turn a passed test red", { timeout: 60_000 }, () => {
+    const result = runFixture(45_435, true);
+
+    expect(result.error).toBeUndefined();
+    expect(result.signal).toBeNull();
+    expect(result.status, diagnostic(result)).toBe(0);
+  });
+
+  it("does not let PGlite teardown turn a failed test green", { timeout: 60_000 }, () => {
+    const result = runFixture(45_436, false);
+
+    expect(result.error).toBeUndefined();
+    expect(result.signal).toBeNull();
+    expect(result.status, diagnostic(result)).toBe(1);
+  });
+});

apps/cloud/test-fixtures/test-globalsetup-exit/fixture.test.ts

@@ -0,0 +1,7 @@
+import { expect, it } from "@effect/vitest";
+
+it("exercises the global-setup exit path", () => {
+  if (process.env.TEST_GLOBALSETUP_SHOULD_PASS === "true") return;
+
+  expect("deliberate failure").toBe("reported as success");
+});

apps/cloud/test-fixtures/test-globalsetup-exit/vitest.config.ts

@@ -0,0 +1,12 @@
+import { resolve } from "node:path";
+import { defineConfig } from "vitest/config";
+
+const appRoot = resolve(__dirname, "../..");
+
+export default defineConfig({
+  root: appRoot,
+  test: {
+    include: ["test-fixtures/test-globalsetup-exit/fixture.test.ts"],
+    globalSetup: [resolve(appRoot, "scripts/test-globalsetup.ts")],
+  },
+});

e2e/package.json

@@ -11,6 +11,7 @@
     "test:selfhost-docker": "vitest run --project selfhost-docker",
     "test:cloudflare": "vitest run --project cloudflare",
     "test:local": "vitest run --project local",
+    "test:ci-shard": "vitest run --config scripts/ci-shard.vitest.config.ts",
     "test:watch": "vitest",
     "ports": "bun scripts/ports.ts",
     "summary": "bun scripts/summary.ts",

e2e/scenarios/artifact-approval.test.ts

@@ -32,11 +32,9 @@ import { Api, Target } from "../src/services";
 const coreApi = composePluginApi([] as const);
 
 /**
- * How long the scenario waits before approving.
- *
- * Long enough to be a real human pause rather than a same-tick round trip, and
- * well past the 4-minute paused-execution lease the MCP plane advertises being
- * irrelevant here. Kept modest so the scenario stays inside the default timeout.
+ * Long enough to cover the observed human pause while staying well inside the
+ * 15-minute approval lease. The shard planner keeps this file isolated so the
+ * real elapsed-time assertion overlaps the rest of CI instead of blocking it.
  */
 const APPROVAL_DELAY_MS = 65_000;
 
@@ -71,7 +69,7 @@ const pausedExecutionId = (structured: unknown): string | undefined =>
   (structured as { readonly executionId?: string } | null)?.executionId;
 
 scenario(
-  "Artifacts · a destructive action approved from an artifact runs, even minutes later",
+  "Artifacts · a destructive action approved from an artifact runs after a human-scale delay",
   { timeout: 240_000 },
   Effect.gen(function* () {
     const target = yield* Target;
@@ -130,8 +128,9 @@ scenario(
         "the action does not run while it is waiting on approval",
       ).toBe(false);
 
-      // The human reads the request and decides. This is the wait that makes the
-      // approval window a real promise rather than a same-request formality.
+      // Exercise the real API and persistence path across an actual delay. A
+      // private fake clock cannot prove that request-scoped engines and durable
+      // storage still agree after the originating request has been gone.
       yield* Effect.sleep(APPROVAL_DELAY_MS);
 
       const approved = yield* client.executions.resume({