fix(cloud): a runaway job on a cloud machine is stopped before the machine stops answering

apps/server/src/environmentControl/E2bProvisionRuntime.test.ts

@@ -9,6 +9,7 @@ import { describe, expect, it } from "vite-plus/test";
 
 import {
   e2bPythonResult,
+  guardMemoryCommand,
   protectEnvdCommand,
   uploadFile,
   warmSealHomePaths,
@@ -162,3 +163,66 @@ describe("protectEnvdCommand", () => {
     }
   });
 });
+
+describe("guardMemoryCommand", () => {
+  it("installs and starts earlyoom once, and leaves a running one with these settings alone", async () => {
+    const root = await NodeFSP.mkdtemp(NodePath.join(NodeOS.tmpdir(), "earlyoom-guard-"));
+    const bin = NodePath.join(root, "bin");
+    const calls = NodePath.join(root, "calls.log");
+    const active = NodePath.join(root, "active");
+    const script = (body: string) => `#!/bin/sh\n${body}\n`;
+    await NodeFSP.mkdir(bin);
+    const tools: Record<string, string> = {
+      sudo: script('[ "$1" = -n ] && shift\nexec "$@"'),
+      // Runs the unit's command in place of systemd's transient service.
+      "systemd-run": script(
+        `printf '%s\\n' "systemd-run $*" >> '${calls}'\nwhile [ "$1" != sh ]; do shift; done\nexec "$@"`,
+      ),
+      "apt-get": script(
+        `echo "apt-get $*" >> '${calls}'\nprintf '#!/bin/sh\\n' > '${bin}/earlyoom'\nchmod +x '${bin}/earlyoom'`,
+      ),
+      systemctl: script(
+        `echo "systemctl $*" >> '${calls}'\ncase "$1" in\n  restart) touch '${active}' ;;\n  is-active) [ -f '${active}' ] ;;\nesac`,
+      ),
+    };
+    for (const [name, body] of Object.entries(tools))
+      await NodeFSP.writeFile(NodePath.join(bin, name), body, { mode: 0o755 });
+    const defaults = NodePath.join(root, "earlyoom");
+    const run = () =>
+      NodeChildProcess.execFileSync("sh", ["-c", guardMemoryCommand(defaults)], {
+        env: { ...process.env, PATH: `${bin}:/usr/bin:/bin` },
+      });
+    const callLog = async () =>
+      (await NodeFSP.readFile(calls, "utf8"))
+        .split("\n")
+        .filter(Boolean)
+        .map((line) => line.split(" ").slice(0, 2).join(" "));
+    try {
+      run();
+      expect(await callLog()).toEqual([
+        "systemd-run --quiet",
+        "apt-get install",
+        "systemctl enable",
+        "systemctl restart",
+      ]);
+      expect(await NodeFSP.readFile(defaults, "utf8")).toBe(
+        'EARLYOOM_ARGS="-m 8 -s 100 -r 0 --avoid (^|/)(envd|systemd|sshd|claude|codex|node)$ --prefer (^|/)(bun|python3|vitest|tsc|esbuild)$"\n',
+      );
+
+      await NodeFSP.writeFile(calls, "");
+      run();
+      expect(await callLog()).toEqual(["systemctl is-active"]);
+
+      await NodeFSP.writeFile(defaults, 'EARLYOOM_ARGS="-m 2"\n');
+      await NodeFSP.writeFile(calls, "");
+      run();
+      expect(await callLog()).toEqual([
+        "systemd-run --quiet",
+        "systemctl enable",
+        "systemctl restart",
+      ]);
+    } finally {
+      await NodeFSP.rm(root, { recursive: true, force: true });
+    }
+  });
+});

apps/server/src/environmentControl/E2bProvisionRuntime.ts

@@ -148,10 +148,37 @@ MemoryLow=256M'
 [ "$(cat ${file} 2>/dev/null)" = "$want" ] || { sudo -n mkdir -p ${shellQuote(directory)} && printf '%s\\n' "$want" | sudo -n tee ${file} > /dev/null && sudo -n systemctl daemon-reload; }`;
 };
 
-/** Why envd could not be protected, or null once it is. Never throws. */
+/**
+ * earlyoom's settings. When free memory falls to 8% it stops the process with the highest score,
+ * preferring test runners and builds, and avoiding envd and the agents. Measured on this template
+ * with 32 bun workers filling memory over a 1.5 GB file corpus: without it envd stopped answering
+ * for half a minute; with it envd answered within a second, earlyoom stopped only bun workers, and
+ * the T3 server and claude kept running. The T3 server runs as `node`, so it is avoided too; a
+ * runaway node test worker is still stopped, as the biggest process left.
+ */
+const EARLYOOM_ARGS =
+  "-m 8 -s 100 -r 0 --avoid (^|/)(envd|systemd|sshd|claude|codex|node)$ --prefer (^|/)(bun|python3|vitest|tsc|esbuild)$";
+
+/**
+ * Keeps one runaway job from freezing the whole box: the kernel only kills a process once memory
+ * is gone, and a box thrashing until then stops answering, envd included. Nothing runs when
+ * earlyoom already runs with these settings; otherwise its install and setup start as a
+ * transient unit, so a slow apt never holds up a wake, and the next wake tries again if it failed.
+ */
+export const guardMemoryCommand = (defaults = "/etc/default/earlyoom") => {
+  const file = shellQuote(defaults);
+  const want = shellQuote(`EARLYOOM_ARGS="${EARLYOOM_ARGS}"`);
+  const setup = `command -v earlyoom >/dev/null || DEBIAN_FRONTEND=noninteractive apt-get install -y -q earlyoom || { apt-get update -q && DEBIAN_FRONTEND=noninteractive apt-get install -y -q earlyoom; }; printf '%s\\n' ${want} > ${file} && systemctl enable earlyoom && systemctl restart earlyoom`;
+  return `{ [ "$(cat ${file} 2>/dev/null)" = ${want} ] && systemctl is-active --quiet earlyoom; } || sudo -n systemd-run --quiet --collect --unit=t3-earlyoom-setup sh -c ${shellQuote(setup)}`;
+};
+
+/** Why envd or the box's memory could not be protected, or null once both are. Never throws. */
 const protectEnvd = (sandbox: Sandbox) =>
   e2bPythonResult(
-    sandbox.commands.run(protectEnvdCommand(), { timeoutMs: 30_000, requestTimeoutMs: 30_000 }),
+    sandbox.commands.run(
+      `{ ${protectEnvdCommand()}\n}; envd=$?; ${guardMemoryCommand()}; memory=$?; exit $((envd | memory))`,
+      { timeoutMs: 30_000, requestTimeoutMs: 30_000 },
+    ),
   )
     .then(({ exitCode, stderr }) => (exitCode === 0 ? null : stderr.trim() || `exit ${exitCode}`))
     .catch((error: unknown) => String(error));