fix(ssh): reconnecting no longer restarts the managed remote server

packages/ssh/src/tunnel.test.ts

@@ -1,4 +1,5 @@
 import { assert, describe, it } from "@effect/vitest";
+import * as NodeNet from "node:net";
 import * as NodeServices from "@effect/platform-node/NodeServices";
 import * as NetService from "@t3tools/shared/Net";
 import * as HostProcess from "@t3tools/shared/HostProcess";
@@ -8,6 +9,7 @@ import * as Effect from "effect/Effect";
 import * as Fiber from "effect/Fiber";
 import * as FileSystem from "effect/FileSystem";
 import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
 import * as Result from "effect/Result";
 import * as Sink from "effect/Sink";
 import * as Stream from "effect/Stream";
@@ -291,7 +293,10 @@ describe("ssh tunnel scripts", () => {
     assert.include(launch, "resolve_default_runtime_port()");
     assert.include(launch, 'DEFAULT_RUNTIME_INFO="$(resolve_default_runtime_port');
     assert.include(launch, "if (!Number.isInteger(pid) || pid <= 0 || !Number.isInteger(port))");
-    assert.include(launch, 'PID_TO_STOP="${REMOTE_PID:-$DEFAULT_RUNTIME_PID}"');
+    assert.include(
+      launch,
+      'if [ -n "$DEFAULT_REMOTE_PORT" ] && [ "$REMOTE_MANAGED" = "managed" ] && [ "$DEFAULT_RUNTIME_PID" = "$REMOTE_PID" ]; then',
+    );
     assert.include(launch, 'REMOTE_PORT="$DEFAULT_REMOTE_PORT"');
     assert.include(launch, 'rm -f "$PID_FILE"');
     assert.include(launch, "printf 'external\\n' >\"$MANAGED_FILE\"");
@@ -687,6 +692,39 @@ describe("ssh tunnel scripts", () => {
   );
 });
 
+// Runs a generated script with an isolated HOME, as an SSH remote would.
+const runScript = (home: string, args: ReadonlyArray<string>) =>
+  Effect.gen(function* () {
+    const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+    const child = yield* spawner.spawn(
+      ChildProcess.make("sh", args, {
+        env: { PATH: process.env.PATH ?? "", HOME: home },
+        extendEnv: false,
+      }),
+    );
+    const [stdout, stderr, exitCode] = yield* Effect.all(
+      [
+        child.stdout.pipe(
+          Stream.decodeText(),
+          Stream.runFold(
+            () => "",
+            (acc, chunk) => acc + chunk,
+          ),
+        ),
+        child.stderr.pipe(
+          Stream.decodeText(),
+          Stream.runFold(
+            () => "",
+            (acc, chunk) => acc + chunk,
+          ),
+        ),
+        child.exitCode.pipe(Effect.map(Number)),
+      ],
+      { concurrency: "unbounded" },
+    );
+    return { stdout, stderr, exitCode };
+  });
+
 // The archive runner is generated shell; string assertions cannot prove the
 // lock excludes concurrent installers. Run the real script against a tiny
 // fake archive served from a file:// mirror.
@@ -696,37 +734,7 @@ describe("archive runner script", () => {
   const windowsHost = hostPlatform === "win32";
   const archiveVersion = "1.2.3-preview.20260911.4";
 
-  const runRunner = (home: string, runner: string) =>
-    Effect.gen(function* () {
-      const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
-      const child = yield* spawner.spawn(
-        ChildProcess.make("sh", [runner, "--version"], {
-          env: { PATH: process.env.PATH ?? "", HOME: home },
-          extendEnv: false,
-        }),
-      );
-      const [stdout, stderr, exitCode] = yield* Effect.all(
-        [
-          child.stdout.pipe(
-            Stream.decodeText(),
-            Stream.runFold(
-              () => "",
-              (acc, chunk) => acc + chunk,
-            ),
-          ),
-          child.stderr.pipe(
-            Stream.decodeText(),
-            Stream.runFold(
-              () => "",
-              (acc, chunk) => acc + chunk,
-            ),
-          ),
-          child.exitCode.pipe(Effect.map(Number)),
-        ],
-        { concurrency: "unbounded" },
-      );
-      return { stdout, stderr, exitCode };
-    });
+  const runRunner = (home: string, runner: string) => runScript(home, [runner, "--version"]);
 
   // A fake "executable" that answers --version, packed the way the release
   // workflow packs the real archive: one top-level directory named after the
@@ -799,3 +807,243 @@ describe("archive runner script", () => {
     60_000,
   );
 });
+
+// Reuse decisions also live in generated shell. Drive the real launch script
+// against a fake server that answers readiness probes and writes the default
+// runtime file the way `t3 serve` does.
+describe("launch script", () => {
+  const windowsHost = HostProcess.Platform.defaultValue() === "win32";
+  const FAKE_SERVER = `const fs = require("node:fs");
+const http = require("node:http");
+const path = require("node:path");
+const args = process.argv.slice(2);
+const flag = (name) => args[args.indexOf(name) + 1];
+// --answer-once stalls every request after the first, like a busy server;
+// --exit-after-answer exits right after the first answer.
+const answerOnce = args.includes("--answer-once");
+const exitAfterAnswer = args.includes("--exit-after-answer");
+let answered = false;
+const server = http.createServer((_request, response) => {
+  if (answerOnce && answered) return;
+  answered = true;
+  response.end("ok", () => {
+    if (exitAfterAnswer) process.exit(0);
+  });
+});
+server.listen(Number(flag("--port")), "127.0.0.1", () => {
+  const port = server.address().port;
+  const userdata = path.join(flag("--base-dir"), "userdata");
+  fs.mkdirSync(userdata, { recursive: true });
+  fs.writeFileSync(
+    path.join(userdata, "server-runtime.json"),
+    JSON.stringify({ version: 1, pid: process.pid, port, origin: "http://127.0.0.1:" + port }),
+  );
+  console.log(port);
+});
+`;
+
+  const isAlive = (pid: number) => {
+    try {
+      process.kill(pid, 0);
+      return true;
+    } catch {
+      return false;
+    }
+  };
+
+  const freeLoopbackPort = Effect.callback<number>((resume) => {
+    const server = NodeNet.createServer();
+    server.listen(0, "127.0.0.1", () => {
+      const address = server.address();
+      server.close(() =>
+        resume(Effect.succeed(typeof address === "object" && address ? address.port : 0)),
+      );
+    });
+  });
+
+  // Whether anything still accepts connections on the port. Unlike a PID
+  // check, this cannot mistake an unreaped, exited server for a running one.
+  const isListening = (port: number) =>
+    Effect.callback<boolean>((resume) => {
+      const socket = NodeNet.connect(port, "127.0.0.1");
+      socket.once("connect", () => {
+        socket.destroy();
+        resume(Effect.succeed(true));
+      });
+      socket.once("error", () => resume(Effect.succeed(false)));
+    });
+
+  const launchResult = (stdout: string) => JSON.parse(stdout.trim().split("\n").at(-1) ?? "");
+
+  // Writes the launch script for a fake server into a temporary HOME. The
+  // seeded remembered port keeps the test off the default 3773, which a live
+  // T3 on this machine may own.
+  const prepareLaunch = Effect.gen(function* () {
+    const fs = yield* FileSystem.FileSystem;
+    const root = yield* fs.makeTempDirectoryScoped({ prefix: "t3-launch-script-" });
+    const home = `${root}/home`;
+    const stateDir = `${home}/.t3/ssh-launch/test-state`;
+    const fakeServer = `${root}/fake-t3.cjs`;
+    const launch = `${root}/launch.sh`;
+    yield* fs.makeDirectory(stateDir, { recursive: true });
+    yield* fs.writeFileString(`${stateDir}/port`, `${yield* freeLoopbackPort}\n`);
+    yield* fs.writeFileString(fakeServer, FAKE_SERVER);
+    yield* fs.writeFileString(
+      launch,
+      SshTunnel.buildRemoteLaunchScript({ nodeScriptPath: fakeServer }),
+    );
+    const readManagedPid = fs
+      .readFileString(`${stateDir}/pid`)
+      .pipe(Effect.map((pid) => Number(pid.trim())));
+    // Launched servers outlive the script, so stop each PID the script
+    // recorded, including one it recorded just before a failed assertion.
+    const launchedPids: Array<number> = [];
+    yield* Effect.addFinalizer(() =>
+      readManagedPid.pipe(
+        Effect.map((pid) => [...launchedPids, pid]),
+        Effect.orElseSucceed(() => launchedPids),
+        Effect.map((pids) => {
+          for (const pid of pids) {
+            if (isAlive(pid)) process.kill(pid);
+          }
+        }),
+      ),
+    );
+    const runLaunch = runScript(home, [launch, "test-state"]).pipe(
+      Effect.map((result) => {
+        assert.equal(result.exitCode, 0, result.stderr);
+        return launchResult(result.stdout);
+      }),
+    );
+    const launchManaged = Effect.gen(function* () {
+      const launched = yield* runLaunch;
+      const pid = yield* readManagedPid;
+      launchedPids.push(pid);
+      assert.equal(launched.serverKind, "managed");
+      return { launched, pid };
+    });
+    // Starts a separate server on the default home, as a service would, and
+    // returns the port it reports. An unreaped server runs under a parent that
+    // never waits on it, so once it exits it lingers as a zombie.
+    const startDefaultHomeServer = (
+      flags: ReadonlyArray<string>,
+      options?: { readonly unreaped?: boolean },
+    ) =>
+      Effect.gen(function* () {
+        const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
+        const args = [fakeServer, "serve", "--port", "0", "--base-dir", `${home}/.t3`, ...flags];
+        const server = yield* spawner.spawn(
+          options?.unreaped
+            ? ChildProcess.make("sh", ["-c", 'node "$@" & exec sleep 600', "sh", ...args])
+            : ChildProcess.make("node", args),
+        );
+        const port = yield* server.stdout.pipe(
+          Stream.decodeText(),
+          Stream.splitLines,
+          Stream.runHead,
+        );
+        return Number(Option.getOrThrow(port));
+      });
+    return { stateDir, readManagedPid, runLaunch, launchManaged, startDefaultHomeServer };
+  });
+
+  it.effect.skipIf(windowsHost)(
+    "reuses its own managed server and yields only to a different default-home server",
+    () =>
+      Effect.gen(function* () {
+        const fs = yield* FileSystem.FileSystem;
+        const { stateDir, readManagedPid, runLaunch, launchManaged, startDefaultHomeServer } =
+          yield* prepareLaunch;
+        const { launched, pid: managedPid } = yield* launchManaged;
+
+        // The managed server wrote the default runtime file itself, so a
+        // reconnect must keep it rather than stop it as an external server.
+        assert.deepEqual(yield* runLaunch, launched);
+        assert.equal(yield* readManagedPid, managedPid);
+        assert.isTrue(isAlive(managedPid));
+
+        // A remembered port that is stale must not hide our own server: the
+        // runtime file names its port.
+        yield* fs.writeFileString(`${stateDir}/port`, `${yield* freeLoopbackPort}\n`);
+        assert.deepEqual(yield* runLaunch, launched);
+        assert.equal(yield* readManagedPid, managedPid);
+        assert.equal(
+          (yield* fs.readFileString(`${stateDir}/port`)).trim(),
+          `${launched.remotePort}`,
+        );
+
+        // A different server on the default home, such as a service, still
+        // takes over from the managed one.
+        const servicePort = yield* startDefaultHomeServer([]);
+        assert.deepEqual(yield* runLaunch, {
+          remotePort: servicePort,
+          serverKind: "external",
+        });
+        assert.isFalse(yield* isListening(launched.remotePort));
+        assert.isFalse(yield* fs.exists(`${stateDir}/pid`));
+      }).pipe(Effect.provide(NodeServices.layer)),
+    60_000,
+  );
+
+  it.effect.skipIf(windowsHost)(
+    "adopts a server it has no managed record for instead of restarting it",
+    () =>
+      Effect.gen(function* () {
+        const fs = yield* FileSystem.FileSystem;
+        const { stateDir, launchManaged, runLaunch } = yield* prepareLaunch;
+        const { launched, pid } = yield* launchManaged;
+
+        // An interrupted launch, or a recycled PID, can leave a PID that matches
+        // the runtime file without the managed marker. A changed runner must not
+        // turn that into a restart of a server this launcher cannot claim.
+        yield* fs.remove(`${stateDir}/managed`);
+        yield* fs.writeFileString(`${stateDir}/run-t3.sh`, "# superseded runner\n");
+        assert.deepEqual(yield* runLaunch, {
+          remotePort: launched.remotePort,
+          serverKind: "external",
+        });
+        assert.isTrue(isAlive(pid));
+      }).pipe(Effect.provide(NodeServices.layer)),
+    60_000,
+  );
+
+  it.effect.skipIf(windowsHost)(
+    "keeps an adopted default-home server without launching another beside it",
+    () =>
+      Effect.gen(function* () {
+        const fs = yield* FileSystem.FileSystem;
+        const { stateDir, runLaunch, startDefaultHomeServer } = yield* prepareLaunch;
+        // Ownership files from an earlier managed launch that lost its PID.
+        yield* fs.writeFileString(`${stateDir}/managed`, "managed\n");
+        const servicePort = yield* startDefaultHomeServer(["--answer-once"]);
+
+        // The server answers adoption and then stalls. A second readiness
+        // probe would launch another server on the same home.
+        assert.deepEqual(yield* runLaunch, { remotePort: servicePort, serverKind: "external" });
+        assert.isFalse(yield* fs.exists(`${stateDir}/pid`));
+      }).pipe(Effect.provide(NodeServices.layer)),
+    60_000,
+  );
+
+  it.effect.skipIf(windowsHost)(
+    "launches a replacement when the adopted server exits during the handover",
+    () =>
+      Effect.gen(function* () {
+        const { readManagedPid, runLaunch, launchManaged, startDefaultHomeServer } =
+          yield* prepareLaunch;
+        const { pid: previousPid } = yield* launchManaged;
+        yield* startDefaultHomeServer(["--exit-after-answer"], { unreaped: true });
+
+        // The service answers adoption and then exits while the managed
+        // server is being stopped, staying a zombie, so the script must launch
+        // a new server rather than return the dead one. The freed port may be
+        // reused.
+        const result = yield* runLaunch;
+        assert.equal(result.serverKind, "managed");
+        const replacementPid = yield* readManagedPid;
+        assert.notEqual(replacementPid, previousPid);
+        assert.isTrue(isAlive(replacementPid));
+      }).pipe(Effect.provide(NodeServices.layer)),
+    60_000,
+  );
+});