fix(cloud): a new cloud chat opens even when its project's setup fails, and says so

apps/server/src/environmentControl/E2bProvisionRuntime.ts

@@ -249,6 +249,7 @@ function guestInput(
       ...extra,
       ...(follow ? { follow } : {}),
       ...(manifest.setup ? { setup: manifest.setup } : {}),
+      ...(manifest.forChat ? { forChat: true } : {}),
     },
     operation.request.agentDriver,
   );

apps/server/src/environmentControl/EnvironmentControl.ts

@@ -73,6 +73,7 @@ import {
   withCurrentSetup,
 } from "./ProvisionPreparation.ts";
 import {
+  isBaseBuild,
   makeSpareClaims,
   makeWarmBaseStore,
   makeWarmBaseUpkeep,
@@ -1434,6 +1435,8 @@ export const layer = Layer.effect(
     const store = yield* ProvisionOperationStore;
     const boxUsage = yield* BoxUsageStore;
     const manifests = makeProvisionPreparationStore(stateDir);
+    const warmStore = makeWarmBaseStore(stateDir);
+    const spareStore = makeWarmBaseStore(stateDir, "spares");
     const legacyLeases = yield* Effect.tryPromise({
       try: () =>
         NodeFSP.readFile(NodePath.join(stateDir, "provisioned-sandbox-leases.json"), "utf8").catch(
@@ -1527,10 +1530,19 @@ export const layer = Layer.effect(
             });
       return result.profile;
     };
-    /** A chat's manifest to prepare it from, with the setup this manager is configured with now. */
+    /**
+     * A manifest to prepare from, with the setup this manager is configured with now, marked
+     * `forChat` unless it is a warm base or spare build.
+     */
     const preparing = async (manifest: ProvisionPreparationManifest) => {
       const manager = await resolve();
-      return manager ? withCurrentSetup(manager.config, manifest) : manifest;
+      const current = manager ? await withCurrentSetup(manager.config, manifest) : manifest;
+      // A build record that cannot be read keeps the strict default rather than failing the open.
+      const build = await isBaseBuild(
+        { warmBases: warmStore, spares: spareStore },
+        manifest.request,
+      ).catch(() => true);
+      return build ? current : { ...current, forChat: true };
     };
     const resolve = () =>
       (async () => {
@@ -2278,7 +2290,6 @@ export const layer = Layer.effect(
         spareClaim,
       );
     };
-    const warmStore = makeWarmBaseStore(stateDir);
     const readyE2bBuild = async (operation: ProvisionOperation) => {
       const manager = await requireManager();
       if (
@@ -2353,7 +2364,6 @@ export const layer = Layer.effect(
           apiKey: (await requireManager()).config.e2bApiKey,
         }).deleteSnapshot(snapshotId),
     });
-    const spareStore = makeWarmBaseStore(stateDir, "spares");
     const spareClaims = makeSpareClaims(stateDir);
     const spares = makeWarmBaseUpkeep({
       ...buildPorts,

apps/server/src/environmentControl/NamespaceMacRuntime.ts

@@ -302,6 +302,7 @@ export function makeNamespaceMacRuntime(config: {
             toolInstall,
             // A builder makes a template, not a chat, so it runs only the build commands.
             ...(!builder && manifest.setup ? { setup: manifest.setup } : {}),
+            ...(!builder && manifest.forChat ? { forChat: true } : {}),
           },
           operation.request.agentDriver,
         ),

apps/server/src/environmentControl/NamespaceProvisionRuntime.ts

@@ -679,6 +679,7 @@ except FileExistsError:
           ...(follow ? { follow } : {}),
           toolInstall,
           ...(manifest.setup ? { setup: manifest.setup } : {}),
+          ...(manifest.forChat ? { forChat: true } : {}),
         },
         operation.request.agentDriver,
       ),

apps/server/src/environmentControl/ProvisionPreparation.ts

@@ -114,6 +114,11 @@ export const ProvisionPreparationManifest = Schema.Struct({
       unreadable: Schema.optional(Schema.Array(Schema.String)),
     }),
   ),
+  /**
+   * Never stored, like `setup`: the manager sets it when it prepares a box for a chat rather than
+   * a warm base or spare build. A chat opens when its setup fails; a build must not.
+   */
+  forChat: Schema.optional(Schema.Boolean),
 });
 export type ProvisionPreparationManifest = typeof ProvisionPreparationManifest.Type;
 const decodeManifest = Schema.decodeUnknownSync(

apps/server/src/environmentControl/remotePreparation.test.ts

@@ -336,7 +336,27 @@ describe("remote preparation subprocess", () => {
     ]);
   });
 
-  it("refuses to call a box ready when its setup failed, on every retry", async () => {
+  it("opens a new chat whose setup fails, running the rest and telling the chat where to look", async () => {
+    const input = {
+      ...(await fixture()),
+      prepareCommands: ["echo deps broke >&2; exit 3", "touch ../ran-after"],
+      forChat: true,
+    };
+    const ready = await prepareRemoteHost(localPort, input);
+    pids.add(ready.serverPid);
+    expect([
+      ready.refreshError,
+      await NodeFSP.readFile(NodePath.join(input.root, "home/.t3/setup-failure.log"), "utf8"),
+      exited(ready.serverPid),
+    ]).toEqual([
+      "Preparation command failed: deps broke",
+      "Preparation command failed: deps broke\n$ echo deps broke >&2; exit 3\n\n",
+      false,
+    ]);
+    await NodeFSP.access(NodePath.join(input.root, "ran-after"));
+  });
+
+  it("refuses to call a warm base or spare build ready when its setup failed, on every retry", async () => {
     const input = { ...(await fixture()), prepareCommands: ["exit 3", "touch ../ran-after"] };
     await expect(prepareRemoteHost(localPort, input)).rejects.toThrow(/Preparation command failed/);
     await expect(prepareRemoteHost(localPort, input)).rejects.toThrow(/Preparation command failed/);

apps/server/src/environmentControl/remotePreparation.ts

@@ -100,12 +100,19 @@ export interface RemotePreparationInput {
         readonly unreadable?: ReadonlyArray<string> | undefined;
       }
     | undefined;
+  /**
+   * The box serves a chat, not a warm base or spare build that others will
+   * share. A failing prepare command is then reported in `refreshError` and
+   * `setup-failure.log` instead of failing preparation. Excluded from the
+   * intent hash like `setup`.
+   */
+  readonly forChat?: boolean | undefined;
 }
 
 export const RemotePreparationReady = Schema.Struct({
   ...ProvisionReadiness.fields,
   headRevision: ProvisionReadiness.fields.t3Revision,
-  /** Why this open could not fetch the followed branch or rerun the setup of a box that served before. */
+  /** Why this open could not fetch the followed branch, or which of a chat's setup commands failed. */
   refreshError: Schema.optional(Schema.NullOr(Schema.String)),
   artifactSha256: ProvisionReadiness.fields.preparationHash,
   runtimeVersion: Schema.String,
@@ -688,7 +695,7 @@ def prepare(spec):
         for value, length in hashes:
             if not re.fullmatch('[0-9a-f]{' + str(length) + '}', value):
                 raise RuntimeError('Expected an exact revision or hash')
-        intent = hashlib.sha256(json.dumps({key: value for key, value in spec.items() if key not in ('artifactSources', 'runtime', 'follow', 'checkOnly', 'toolInstall', 'prepareEnvironment', 'setup')}, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
+        intent = hashlib.sha256(json.dumps({key: value for key, value in spec.items() if key not in ('artifactSources', 'runtime', 'follow', 'checkOnly', 'toolInstall', 'prepareEnvironment', 'setup', 'forChat')}, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
         journal_path = root / 'preparation.json'
         adopted = False
         if journal_path.exists():
@@ -1086,24 +1093,24 @@ def prepare(spec):
                 for secret in secrets:
                     text = text.replace(secret, '***')
                 return text
-            # A chat that finished preparing once has a checkout it works in.
-            # Its setup failing again on the chat's own branch is reported, not
-            # allowed to keep it off a new build, and the commands after it
-            # still run so the services they start come up. A root prepared
-            # before the journal recorded this shows it by a server having
-            # started, or by the chat database a snapshot carries.
-            prepared = journal.get('prepared') or server_path.exists() or (t3home / 'userdata' / 'statev2.sqlite').exists()
+            # A chat's setup failing is reported, not allowed to keep the chat
+            # closed or off a new build, and the commands after it still run so
+            # the services they start come up. Only a base that a snapshot or a
+            # spare will share, which never sends forChat, fails on it. A root
+            # that already served a chat is a chat's too, known by its journal,
+            # a server having started, or the chat database a snapshot carries.
+            tolerated = spec.get('forChat') is True or journal.get('prepared') or server_path.exists() or (t3home / 'userdata' / 'statev2.sqlite').exists()
             with step('prepareCommands'):
                 for index, command_line in enumerate(prepare):
                     try:
                         with step('prepareCommand.' + str(index)):
                             run(['sh', '-lc', command_line], project, prepare_env, timeout=PREPARE_COMMAND_SECONDS)
                     except RuntimeError as error:
                         failure = redact(str(error))
-                        if not prepared:
+                        if not tolerated:
                             raise RuntimeError(failure) from None
                         # A timeout is usually an outage the later commands
-                        # would each wait out too, holding a reopen for hours.
+                        # would each wait out too, holding an open for hours.
                         later = len(prepare) - index - 1
                         if isinstance(error, CommandTimeout) and later:
                             failure += '\nSkipped the ' + str(later) + ' setup command' + ('s' if later > 1 else '') + ' after it'

apps/server/src/environmentControl/warmBases.test.ts

@@ -7,6 +7,7 @@ import * as DateTime from "effect/DateTime";
 import * as Schema from "effect/Schema";
 import { describe, expect, it } from "vite-plus/test";
 import {
+  isBaseBuild,
   makeSpareClaims,
   makeWarmBaseStore,
   makeWarmBaseUpkeep,
@@ -752,3 +753,34 @@ describe("a repository with a warm base and a spare", () => {
     }
   });
 });
+
+describe("telling a base build from a chat", () => {
+  it("knows each repository's build in progress by its request, on its own provider", async () => {
+    const directory = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "t3-base-build-"));
+    try {
+      const stores = {
+        warmBases: makeWarmBaseStore(directory),
+        spares: makeWarmBaseStore(directory, "spares"),
+      };
+      await stores.spares.write(
+        record({ build: { key: KEY, requestId: buildId, startedAt: iso(NOW) } }),
+      );
+      const chatId = ProvisionRequestId.make("11111111-1111-4111-a111-000000000002");
+      const request = (provider: "e2b" | "namespace", requestId: ProvisionRequestId) => ({
+        provider,
+        requestId,
+        repository: "example/repo",
+      });
+      expect(
+        await Promise.all([
+          isBaseBuild(stores, request("namespace", buildId)),
+          isBaseBuild(stores, request("namespace", chatId)),
+          isBaseBuild(stores, request("e2b", buildId)),
+          isBaseBuild(stores, { provider: "namespace", requestId: buildId }),
+        ]),
+      ).toEqual([true, false, false, false]);
+    } finally {
+      await NodeFSP.rm(directory, { recursive: true, force: true });
+    }
+  });
+});

apps/server/src/environmentControl/warmBases.ts

@@ -340,6 +340,19 @@ export function makeWarmBaseStore(stateDir: string, kind: "warm-bases" | "spares
 }
 export type WarmBaseStore = ReturnType<typeof makeWarmBaseStore>;
 
+/**
+ * Whether a request is its repository's warm base (E2B) or spare (Namespace) build in progress.
+ * Snapshots and chats share what a build leaves, so unlike a chat it must not open on a failed setup.
+ */
+export async function isBaseBuild(
+  stores: { readonly warmBases: WarmBaseStore; readonly spares: WarmBaseStore },
+  request: Pick<ProvisionOperation["request"], "provider" | "requestId" | "repository">,
+) {
+  if (!request.repository) return false;
+  const store = request.provider === "e2b" ? stores.warmBases : stores.spares;
+  return (await store.read(request.repository))?.build?.requestId === request.requestId;
+}
+
 const SpareClaim = Schema.Struct({ by: Schema.String, at: IsoDateTime });
 const decodeClaim = Schema.decodeUnknownSync(Schema.fromJsonString(SpareClaim));
 /** The claimant the upkeep uses to take a spare out of the pool before disposing it. */

apps/server/src/orchestration-v2/DiskHeadroom.ts

@@ -44,7 +44,7 @@ export interface CloudMachine {
   readonly diskBytes: number;
   /** Whether t3_fork_run can copy this machine. */
   readonly forks: boolean;
-  /** The log a reopen whose project setup failed left, and its first line. */
+  /** The log a preparation whose project setup failed left, and its first line. */
   readonly setupFailure?: { readonly log: string; readonly summary: string } | undefined;
 }
 
@@ -92,7 +92,7 @@ export const cloudMachineNoteFor = (input: {
     toldBySession.set(input.session, told.add(input.nativeThreadId));
     const forks = input.mcpTools && (yield* HostProcessPlatform) !== "darwin";
     const t3home = (yield* HostProcessEnvironment).T3CODE_HOME;
-    // Written by the box's preparation when a reopen's project setup fails, removed when it passes.
+    // Written by the box's preparation when its project setup fails, removed when it passes.
     const log = t3home ? NodePath.join(t3home, "setup-failure.log") : undefined;
     const setupFailure = log
       ? yield* Effect.tryPromise(() => NodeFSP.readFile(log, "utf8")).pipe(

docs/operations/cloud-provisioning.md

@@ -103,7 +103,7 @@ It needs `nsc login` on the machine running it, and it disposes what it made and
 
 A woken box that speaks the manager's orchestration protocol connects on the build it runs; only an incompatible one moves to the pinned build before a client connects. A box behind the pinned build moves to it when the reaper finds it idle and about to sleep, at most twice per build, so its next wake needs no update. While any client reports its user present, the manager wakes every paused box that holds an unsettled chat, most recently active first and at most 20 at a time, and renews them while the reports continue. A Mac wakes one at a time, and only while one of the account's four Mac slots stays free. A refused wake retries after 30 seconds, then 1, 2, 4 and 8 minutes, then every 10 minutes. E2B's `autoResume` stays off: clients dial a box's public address directly, so any stray request would wake a box the manager believes is asleep and its six-hour timeout would keep it running unseen.
 
-Every preparation of an existing chat, including an update and the restart of a box whose server died, reruns the repository's prepare commands as the manager is configured now, with `provisioning.shellEnvironment` minus any agent login in their environment. Editing them therefore reaches existing chats on their next preparation. Workspace and home files stay as the chat was created with them, because the checkout is the chat's own. Once a chat has prepared successfully, a failing prepare command no longer stops its update. The commands after it still run, the manager logs the failure as a warning, and the new build starts anyway. The failure is also written to `setup-failure.log` in the chat's T3 home, which the chat's agent is pointed at. A command that times out skips the commands after it, so an outage cannot hold an update for each one. A `shellEnvironment` source that cannot be read leaves that variable out and is reported by name the same way. Reported output shows `***` in place of any `shellEnvironment` value of 8 or more characters. A new chat still fails when its setup fails.
+Every preparation of an existing chat, including an update and the restart of a box whose server died, reruns the repository's prepare commands as the manager is configured now, with `provisioning.shellEnvironment` minus any agent login in their environment. Editing them therefore reaches existing chats on their next preparation. Workspace and home files stay as the chat was created with them, because the checkout is the chat's own. A failing prepare command never keeps a chat from opening or from an update. The commands after it still run, the manager logs the failure as a warning, and the chat's server starts anyway. The failure is also written to `setup-failure.log` in the chat's T3 home, which the chat's agent is pointed at. A command that times out skips the commands after it, so an outage cannot hold an open for each one. A `shellEnvironment` source that cannot be read leaves that variable out and is reported by name the same way. Reported output shows `***` in place of any `shellEnvironment` value of 8 or more characters. Only a warm base, spare or Mac template build fails when its setup fails, so later chats never start from a broken setup.
 
 Before a box's T3 server starts, its preparation sets aside a 256 MB reserve file on Linux. A watcher gives the reserve back and clears package caches when free space drops under 1 GB, so the server can still save the chat. Agents are told at the start of each turn when their disk has under 10 GB free, and a top-level chat's agent is pointed at `t3_fork_run` (E2B, and only with a provider that has MCP tools) and S3, and told to remove worktrees, installs and `/tmp` data it no longer needs, on the first turn of each provider session (a new chat, and again after a wake, restart, account switch or 30 idle minutes), so a long-running chat hears it too.