Speed up and stabilize CI e2e (#1551)

* Speed up selfhost e2e: tunable sandbox deadline + 3-way sharding

* Fix the local e2e suite: toolkit MCP DB lock, Bun-only spawn, stale auth selector

* Stabilize CI process lifecycle

Author: RhysSullivan

.github/workflows/ci.yml

@@ -192,14 +192,25 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          # Each cloud shard boots its own fresh dev stack. On 4 vCPU runners,
-          # four fatter shards keep the longest shard below selfhost while saving
-          # four runner boots and four warm cache restores.
-          - { target: cloud, shard: 1/4, shard-name: 1of4 }
-          - { target: cloud, shard: 2/4, shard-name: 2of4 }
-          - { target: cloud, shard: 3/4, shard-name: 3of4 }
-          - { target: cloud, shard: 4/4, shard-name: 4of4 }
-          - target: selfhost
+          # PGlite is deliberately single-connection, and under a sustained
+          # multi-minute shard it can stop accepting postgres sockets. Keep
+          # every hermetic dev stack short: eight serial shards remove that
+          # lifetime-dependent failure and put cloud below the selfhost lane.
+          - { target: cloud, shard: 1/8, shard-name: 1of8 }
+          - { target: cloud, shard: 2/8, shard-name: 2of8 }
+          - { target: cloud, shard: 3/8, shard-name: 3of8 }
+          - { target: cloud, shard: 4/8, shard-name: 4of8 }
+          - { target: cloud, shard: 5/8, shard-name: 5of8 }
+          - { target: cloud, shard: 6/8, shard-name: 6of8 }
+          - { target: cloud, shard: 7/8, shard-name: 7of8 }
+          - { target: cloud, shard: 8/8, shard-name: 8of8 }
+          # Selfhost shards the same way: each shard is its own runner booting
+          # its own fresh instance (own port block + data dir), so the
+          # project's shared-bootstrap-admin assumption stays intact per shard
+          # and `fileParallelism: false` still serializes within a shard.
+          - { target: selfhost, shard: 1/3, shard-name: 1of3 }
+          - { target: selfhost, shard: 2/3, shard-name: 2of3 }
+          - { target: selfhost, shard: 3/3, shard-name: 3of3 }
     runs-on: blacksmith-4vcpu-ubuntu-2404
     timeout-minutes: 30
     steps:
@@ -241,20 +252,20 @@ jobs:
 
       # The globalsetup boots the target's own dev server (ports are claimed
       # per checkout, so this is hermetic) and tears it down after the run.
-      # --retry=2: browser scenarios can still hit isolated waitFor timeouts
-      # (single-test waitFor timeouts, not systemic failures); a retry on the
-      # same booted stack clears them.
+      # Do not retry scenarios: retries hide flakes and multiply slow timeout
+      # failures. The fixtures and process lifecycle are deterministic enough
+      # that the first result is the result.
       - name: Run cloud scenarios
         if: matrix.target == 'cloud'
         env:
           MCP_SESSION_TIMEOUT_MS: "3000"
           MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS: "6000"
-        run: bunx vitest run --project cloud --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
+        run: bunx vitest run --project cloud ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
         working-directory: e2e
 
       - name: Run selfhost scenarios
         if: matrix.target == 'selfhost'
-        run: bunx vitest run --project selfhost --retry=2
+        run: bunx vitest run --project selfhost ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }}
         working-directory: e2e
 
       # Failed runs keep their trace.zip / session.mp4 / step screenshots in
@@ -268,10 +279,7 @@ jobs:
           retention-days: 7
 
   e2e-local:
-    name: E2E (stdio MCP)
-    # Skipped on pull_request: the local scenario boots a real `executor web`
-    # plus a browser and is currently flaky on PRs. Still runs on push to main.
-    if: github.event_name != 'pull_request'
+    name: E2E (local)
     runs-on: blacksmith-4vcpu-ubuntu-2404
     timeout-minutes: 20
     steps:
@@ -314,15 +322,10 @@ jobs:
         run: bunx playwright install --with-deps chromium chromium-headless-shell
         working-directory: e2e
 
-      # The `local` project is excluded from the default `test` chain (each
-      # scenario boots its own `executor web`). Run just the stdio MCP scenario
-      # here: it is the auto-connect / env-as-secret regression guard, and
-      # running it alone avoids the boot-resource accumulation and the
-      # pre-existing browser flakiness of the rest of the local suite. Expanding
-      # to the full `local` project (bun run test:local) is a follow-up once
-      # those are stabilized.
-      - name: Run the stdio MCP scenario
-        run: bunx vitest run --project local local/stdio-mcp.test.ts
+      # Each scenario owns its server, browser, data directory, and descendants;
+      # run the complete hermetic suite on PRs without scenario retries.
+      - name: Run local scenarios
+        run: bunx vitest run --project local
         working-directory: e2e
 
   desktop-smoke:

apps/cli/src/main.ts

@@ -215,9 +215,11 @@ const waitForShutdownSignal = () =>
     const shutdown = () => resume(Effect.void);
     process.once("SIGINT", shutdown);
     process.once("SIGTERM", shutdown);
+    process.once("SIGHUP", shutdown);
     return Effect.sync(() => {
       process.off("SIGINT", shutdown);
       process.off("SIGTERM", shutdown);
+      process.off("SIGHUP", shutdown);
     });
   });
 

apps/host-selfhost/src/config.ts

@@ -45,6 +45,14 @@ export interface SelfHostConfig {
   readonly organizationName: string;
   /** URL slug for org-prefixed console paths (`/<slug>/policies`). */
   readonly orgSlug: string;
+  /**
+   * Sandbox execution budget passed to the QuickJS runtime, or undefined for
+   * the runtime's own default (5 minutes). An operator knob in principle, but
+   * its real consumer is the e2e harness, which shrinks it to seconds so the
+   * sandbox-deadline scenario proves its race without waiting out real
+   * minutes (the same pattern as MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS on cloud).
+   */
+  readonly sandboxTimeoutMs: number | undefined;
 }
 
 export const resolveDataDir = (): string =>
@@ -151,9 +159,26 @@ export const loadConfig = (): SelfHostConfig => {
     bootstrapAdminName: process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME ?? "Admin",
     organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default",
     orgSlug: resolveOrgSlug(),
+    sandboxTimeoutMs: resolveSandboxTimeoutMs(),
   };
 };
 
+// A malformed value is refused rather than silently ignored: an operator who
+// sets the knob and typos it should find out at boot, not by watching a
+// runaway execution use the 5-minute default.
+const resolveSandboxTimeoutMs = (): number | undefined => {
+  const raw = process.env.EXECUTOR_SANDBOX_TIMEOUT_MS;
+  if (!raw) return undefined;
+  const parsed = Number(raw);
+  if (!Number.isFinite(parsed) || parsed <= 0) {
+    // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob
+    throw new Error(
+      `EXECUTOR_SANDBOX_TIMEOUT_MS ${JSON.stringify(raw)} is not a positive number of milliseconds`,
+    );
+  }
+  return Math.floor(parsed);
+};
+
 // The org slug doubles as a URL segment (`/<slug>/policies`), so an
 // operator-set value must fit the shared grammar and avoid reserved root
 // segments (api, mcp, login, …) — a colliding slug would shadow real routes.

apps/host-selfhost/src/execution.ts

@@ -65,7 +65,12 @@ export const SelfHostHostConfig: Layer.Layer<HostConfig> = Layer.sync(HostConfig
 
 export const SelfHostCodeExecutorProvider: Layer.Layer<CodeExecutorProvider> = Layer.sync(
   CodeExecutorProvider,
-  () => makeQuickJsExecutor(),
+  () => {
+    const { sandboxTimeoutMs } = loadConfig();
+    return makeQuickJsExecutor(
+      sandboxTimeoutMs === undefined ? {} : { timeoutMs: sandboxTimeoutMs },
+    );
+  },
 );
 
 /**

apps/local/src/executor.ts

@@ -19,7 +19,7 @@ import type { McpPluginExtension } from "@executor-js/plugin-mcp";
 import executorConfig from "../executor.config";
 import { localAnalytics } from "./analytics";
 import { localDataMigrations } from "./db/data-migrations";
-import { openOwnedLocalDatabase } from "./db/owned-database";
+import { openOwnedLocalDatabase, type OwnedLocalDatabase } from "./db/owned-database";
 
 interface ResolvedStorage {
   readonly dataDir: string;
@@ -56,6 +56,16 @@ type LocalPlugins = readonly AnyPlugin[];
 
 export interface LocalExecutorOptions {
   readonly activeToolkitSlug?: string;
+  /**
+   * Reuse an already-open owned database instead of opening (and locking) the
+   * data dir again. A toolkit-scoped MCP session differs from the default one
+   * only in its plugin set, so it must ride the running server's DB handle:
+   * `openOwnedLocalDatabase` takes an EXCLUSIVE lock, and a second open from
+   * inside the same process contends with the lock this process already holds.
+   * The borrowed handle is NOT closed when the derived executor disposes —
+   * whoever opened it still owns its lifetime.
+   */
+  readonly borrowedDb?: OwnedLocalDatabase;
 }
 
 const loadLocalPlugins = (options: LocalExecutorOptions = {}) =>
@@ -92,6 +102,10 @@ const loadLocalPlugins = (options: LocalExecutorOptions = {}) =>
 interface LocalExecutorBundle {
   readonly executor: Executor<LocalPlugins>;
   readonly plugins: LocalPlugins;
+  /** The owned DB this bundle opened (or borrowed). Surfaced so a
+   *  toolkit-scoped executor can ride the SAME handle instead of contending
+   *  with this process's own exclusive data-dir lock. */
+  readonly db: OwnedLocalDatabase;
   /** Where this daemon's web UI is reachable, resolved once at boot. Surfaced
    *  so callers building user-facing links (MCP artifact deep links) use the
    *  same origin the executor itself was configured with. */
@@ -151,23 +165,27 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => {
       const tenantId = makeTenantId(cwd);
       const tables = collectTables();
 
-      const owned = yield* Effect.acquireRelease(
-        Effect.tryPromise({
-          try: () =>
-            openOwnedLocalDatabase({
-              dataDir: storage.dataDir,
-              tables,
-              namespace: localNamespace,
-              tenantId,
+      // A borrowed handle is owned by its opener, so it is used as-is and left
+      // open on release; only a handle opened here is closed here.
+      const owned = options.borrowedDb
+        ? options.borrowedDb
+        : yield* Effect.acquireRelease(
+            Effect.tryPromise({
+              try: () =>
+                openOwnedLocalDatabase({
+                  dataDir: storage.dataDir,
+                  tables,
+                  namespace: localNamespace,
+                  tenantId,
+                }),
+              catch: (cause) =>
+                new LocalExecutorCreateError({
+                  message: CREATE_SQLITE_ERROR_MESSAGE,
+                  cause,
+                }),
             }),
-          catch: (cause) =>
-            new LocalExecutorCreateError({
-              message: CREATE_SQLITE_ERROR_MESSAGE,
-              cause,
-            }),
-        }),
-        (database) => Effect.promise(() => database.close()).pipe(Effect.ignore),
-      );
+            (database) => Effect.promise(() => database.close()).pipe(Effect.ignore),
+          );
       const sqlite = owned.db;
       const migration = owned.migration;
 
@@ -243,7 +261,7 @@ const createLocalExecutorLayer = (options: LocalExecutorOptions = {}) => {
           );
       }
 
-      return { executor, plugins, webBaseUrl };
+      return { executor, plugins, webBaseUrl, db: owned };
     }),
   );
 };
@@ -257,6 +275,7 @@ export const createExecutorHandle = async (options: LocalExecutorOptions = {}) =
     executor: bundle.executor,
     plugins: bundle.plugins,
     webBaseUrl: bundle.webBaseUrl,
+    db: bundle.db,
     dispose: async () => {
       await Effect.runPromise(Effect.ignore(bundle.executor.close()));
       await ignorePromiseFailure("disposeRuntime", () => runtime.dispose());

apps/local/src/main.ts

@@ -137,8 +137,13 @@ export const createServerHandlers = async (token: string): Promise<ServerHandler
             },
           };
         }
+        // Borrow the running server's DB handle: this process already holds the
+        // data dir's exclusive ownership lock, so opening it a second time here
+        // fails against ourselves. The toolkit executor differs only in its
+        // plugin set, and the borrowed handle stays open when it disposes.
         const handle = await createExecutorHandle({
           activeToolkitSlug: resource.slug,
+          borrowedDb: (await getExecutorBundle()).db,
         });
         const toolkitEngine = withExecutionAnalytics(
           createExecutionEngine({

apps/local/src/serve.ts

@@ -113,6 +113,8 @@ interface ViteChild {
   readonly stop: () => Promise<void>;
 }
 
+const viteChildSignals = ["SIGINT", "SIGTERM", "SIGHUP"] as const;
+
 async function allocatePort(): Promise<number> {
   const probe = Bun.serve({
     port: 0,
@@ -127,15 +129,15 @@ async function allocatePort(): Promise<number> {
 async function startViteChild(): Promise<ViteChild> {
   const vitePort = await allocatePort();
   const cwd = resolve(import.meta.dirname, "..");
+  const viteEntrypoint = resolve(cwd, "node_modules/vite/bin/vite.js");
   const env = { ...process.env };
   delete env.PORT;
-  // `bunx --bun vite` runs vite under Bun, matching the `dev:vite` script
-  // already in apps/local. --strictPort keeps the URL we hand back stable.
+  // Run Vite directly under Bun, matching the `dev:vite` script without a
+  // bunx wrapper that can outlive its child. --strictPort keeps the URL stable.
   const child: Subprocess = Bun.spawn(
     [
-      "bunx",
-      "--bun",
-      "vite",
+      process.execPath,
+      viteEntrypoint,
       "dev",
       "--port",
       String(vitePort),
@@ -158,33 +160,59 @@ async function startViteChild(): Promise<ViteChild> {
     },
   );
 
+  let stopping = false;
+  const stop = async (): Promise<void> => {
+    if (stopping) {
+      await child.exited;
+      return;
+    }
+    stopping = true;
+    for (const signal of viteChildSignals) process.off(signal, stopOnParentSignal);
+    if (child.exitCode === null) child.kill();
+    await Promise.race([child.exited, Bun.sleep(5_000)]);
+    if (child.exitCode === null) child.kill("SIGKILL");
+    await child.exited;
+  };
+  const stopOnParentSignal = (): void => {
+    // A PTY/session teardown can signal the CLI while Vite is still optimizing
+    // dependencies, before the server's normal stop handle exists. Reap the
+    // owned child immediately; the CLI's signal waiter performs full cleanup
+    // once startup has completed.
+    void stop();
+  };
+  for (const signal of viteChildSignals) process.once(signal, stopOnParentSignal);
+
   const url = `http://127.0.0.1:${vitePort}`;
   const deadline = Date.now() + 30_000;
   while (Date.now() < deadline) {
     // oxlint-disable-next-line executor/no-try-catch-or-throw -- boundary: probing a child process that may not be listening yet
     try {
-      const r = await fetch(`${url}/`, { redirect: "manual" });
+      const r = await fetch(`${url}/`, {
+        redirect: "manual",
+        // A listening socket is not proof that Vite can answer. Bound each
+        // probe so one accepted-but-stalled request cannot defeat the 30s boot
+        // deadline and wedge the entire local e2e suite.
+        signal: AbortSignal.timeout(5_000),
+      });
       if (r.status < 500) {
         await r.body?.cancel();
         return {
           url,
-          stop: async () => {
-            child.kill();
-            await child.exited;
-          },
+          stop,
         };
       }
       await r.body?.cancel();
     } catch {
       // not up yet
     }
     if (child.exitCode !== null) {
+      await stop();
       // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: child process aborted before becoming ready
       throw new Error(`vite dev exited with code ${child.exitCode} before becoming ready`);
     }
     await Bun.sleep(150);
   }
-  child.kill();
+  await stop();
   // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: vite never became reachable
   throw new Error(`vite dev did not become reachable on ${url} within 30s`);
 }

e2e/local/auth.test.ts

@@ -36,8 +36,10 @@ scenario(
           await page.goto(url, { waitUntil: "domcontentloaded" });
           await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });
           // Integrations actually LOAD (the built-in Executor integration) — proves
-          // auth + data, not just the static shell.
-          await page.getByText("built-in").first().waitFor({ timeout: 30_000 });
+          // auth + data, not just the static shell. Matched on the row's stable
+          // testid: the list renders each integration's name + slug, never the
+          // literal "built-in" (that string is only an internal `kind`).
+          await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 });
           // The token is moved out of the URL and persisted to localStorage.
           expect(new URL(page.url()).searchParams.has("_token")).toBe(false);
           const stored = await page.evaluate(() => localStorage.getItem("executor.authToken"));
@@ -70,7 +72,7 @@ scenario(
           await page.getByRole("button", { name: "Connect" }).click();
           await page.getByRole("link", { name: "Secrets" }).first().waitFor({ timeout: 30_000 });
           // The reconnect fully restores — integrations LOAD, not a stale 401.
-          await page.getByText("built-in").first().waitFor({ timeout: 30_000 });
+          await page.getByTestId("integration-entry-executor").first().waitFor({ timeout: 30_000 });
         });
       }),
     );