ericcaiwx-star · Oct 9, 2026 · Oct 10, 2026
@@ -1,7 +1,7 @@
 ---
 created: 2026-09-04
-updated: 2026-10-05
-last_updated_session: authoring-run/5808365b-c590-4c11-92d6-4ee32efc3626
+updated: 2026-10-10
+last_updated_session: authoring-run/c7f52e93-d02d-497c-b146-43640cef1418
 ---
 
 # GitHub Actions testing flow
@@ -102,17 +102,17 @@ after capture; local callers still clean up using the failed run's state file.
 Collection preserves the original error even if observation fails or times out.
 The [CI guide](../testing/ci.md) describes the retained evidence.
 
-Tests delete Agent namespaces and their events before file exit. While each file
-runs, `scripts/ci/run-tests.mjs:runFile` watches Compute-managed Pods and events
-in ready k3d clusters; `scripts/ci/k3d-diagnostics.mjs:projectAgentNamespaceActivity`
-appends Pod transitions and those namespaces' events to the report under
-`agentNamespaces` on pass or failure. Each file retains at most 200 Pod and 200
-event records, and the report retains 40 files. Messages are redacted and
-truncated, and Pod specs dropped. Each file streams raw watches to its own files
-in the cluster directory and removes them when done. Watches are cluster-wide, so
-under `fileConcurrency` a record can include a sibling's namespaces. A failed
-start logs `Agent namespace activity unavailable`. Each lane that writes the
-artifact uploads it.
+Tests delete namespaces and events before exit. `scripts/ci/run-tests.mjs:runFile`
+calls `scripts/ci/k3d-diagnostics.mjs:startAgentNamespaceCapture` before each file.
+It requests initial Pod/Event streams and waits up to 10 seconds per watch for
+the initial-state bookmark. Watch failure stops observers, removes streams and
+logs `Agent namespace activity unavailable`. The independent log directory and
+`finish` still retain failed-wait logs and snapshots.
+`projectAgentNamespaceActivity` retains transitions and namespace events under
+`agentNamespaces` on pass or failure: 200 Pod and 200 event records per file,
+40 files per report. Messages are redacted/truncated; Pod specs dropped. Raw
+per-file streams are removed after projection. Cluster-wide watches can include
+concurrent siblings' namespaces. Lanes upload any resulting artifact.
 
 Dedicated Codex preparation and the operator's offline profile generator share
 `scripts/lib/codex-seccomp-profile.mjs:deriveCodexBwrapProfile`. Preparation
@@ -192,6 +192,10 @@ Per-file cleanup releases its disposable database; job cleanup removes only stat
 
 ## Changelog
 
+- 2026-10-10 09:04: Preserve failed-wait container evidence when optional watch startup fails. (authoring-run/c7f52e93-d02d-497c-b146-43640cef1418 - acb84f827e22218bdb8432c48cbb22d644503cfb)
+
+- 2026-10-10 06:28: Await initial Agent watch synchronization before each test file. (authoring-run/1daba808-076b-48c2-a2ef-79a5f6149f31 - 792304b6a09b5df4f46e3889066f231610b294cf)
+
 - 2026-10-06 02:00: Per-file Agent namespace watch files. (audit-followup-ci-runner)
 
 - 2026-10-04 03:44: Add a non-required affected-package advisory to the accompanying CI change. (authoring-run/5808365b-c590-4c11-92d6-4ee32efc3626 - 070147565f45e720918de9e649b93cad71820b07)

@@ -101,10 +101,12 @@ kubeconfig, environment values and Pod specs are excluded. After a failed prepar
 run, local callers must run `node scripts/ci/cleanup.mjs --state <state-file>`.
 Diagnostics explain setup failures without establishing coverage.
 
-In k3d lanes the runner gives each file a private `OPENCLAW_CI_CONTAINER_LOG_DIR`.
-A test that follows a container with `tests/helpers/container-log-capture.mjs`
-writes a record there only when a wait fails: its markers, Pod and event
-snapshots, and the log, waiting up to 60 s for the container to exit.
+Before each k3d file, the runner awaits initial Pod/Event synchronization
+([flow](../flows/github-actions-testing.md#2-prepare-resources-under-the-job-owner)).
+The independent private `OPENCLAW_CI_CONTAINER_LOG_DIR` remains available when
+watch startup fails. On failed waits,
+`tests/helpers/container-log-capture.mjs` records markers, Pod/event snapshots
+and logs, waiting up to 60 s for container exit.
 `scripts/ci/k3d-diagnostics.mjs:projectContainerLog` keeps at most 1,500 lines,
 redacts environment values and secret shapes in lines and event messages, and
 adds the record to the same report under `containerLogs`. The platform recovery

@@ -302,6 +302,7 @@ export async function captureK3dDiagnostics({ execFile, cluster, lane, statePath
 
 const AGENT_POD_SELECTOR = "app.kubernetes.io/managed-by=openclaw-enterprise";
 const WATCH_SECONDS = 3 * 60 * 60;
+const WATCH_START_TIMEOUT_MS = 10_000;
 const MAX_WATCH_BYTES = 32 * 1024 * 1024;
 const MAX_ACTIVITY_RECORDS = 200;
 const MAX_ACTIVITY_FILES = 40;
@@ -583,6 +584,25 @@ async function stopWatch(child) {
   clearTimeout(timer);
 }
 
+async function waitForInitialWatch(child, path) {
+  const deadline = Date.now() + WATCH_START_TIMEOUT_MS;
+  while (Date.now() < deadline) {
+    if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) {
+      throw new Error("Agent namespace watch could not start.");
+    }
+    for (const { type, object } of watchLines(await readWatchTail(path))) {
+      if (
+        type === "BOOKMARK" &&
+        object.metadata.annotations?.["k8s.io/initial-events-end"] === "true"
+      ) {
+        return;
+      }
+    }
+    await new Promise((resolve) => setTimeout(resolve, 50));
+  }
+  throw new Error("Agent namespace watch startup timed out.");
+}
+
 /**
  * Watches Agent Pods and Kubernetes events in each prepared k3d cluster while
  * one test file runs. Tests delete their namespaces, and their events with
@@ -618,58 +638,57 @@ export async function startAgentNamespaceCapture({ statePath, lane, file }) {
   if (clusters.length === 0) {
     return undefined;
   }
-  const watches = [];
-  for (const cluster of clusters) {
-    const kubectl = cluster.kubectl ?? process.env.OCC_KUBECTL_BIN ?? "kubectl";
-    const scope = ["--kubeconfig", cluster.kubeconfig, "--context", cluster.context];
-    const capture = randomUUID();
-    const paths = {
-      events: join(cluster.directory, `agent-activity-${capture}-events.ndjson`),
-      pods: join(cluster.directory, `agent-activity-${capture}-pods.ndjson`),
-    };
-    const children = [];
-    for (const [kind, path] of Object.entries(paths)) {
-      const query =
-        kind === "pods"
-          ? `/api/v1/pods?watch=true&timeoutSeconds=${WATCH_SECONDS}&labelSelector=${encodeURIComponent(AGENT_POD_SELECTOR)}`
-          : `/api/v1/events?watch=true&timeoutSeconds=${WATCH_SECONDS}`;
-      // The raw watch streams one JSON object per line straight to the cluster's
-      // private directory, which cleanup removes; only the projection is kept.
-      let output;
-      try {
-        output = await open(path, "w", 0o600);
-      } catch (error) {
-        // The caller gets no finish to call: stop the watches already started, whose
-        // running child processes would otherwise keep the runner alive, and drop
-        // their streams.
-        const started = [...watches, { paths, children }];
-        await Promise.all(started.flatMap((watch) => watch.children).map(stopWatch));
-        await Promise.all(
-          started
-            .flatMap((watch) => Object.values(watch.paths))
-            .map((stream) => rm(stream, { force: true })),
-        );
-        throw error;
-      }
-      try {
-        const child = spawn(kubectl, [...scope, "get", "--raw", query], {
-          stdio: ["ignore", output.fd, "ignore"],
-        });
-        child.on("error", () => {});
-        children.push(child);
-      } finally {
-        await output.close();
-      }
-    }
-    watches.push({ cluster, paths, children });
-  }
-  // Inside the first cluster's private directory, which cleanup removes.
+  // Container logs are independent of the optional watches. A stalled observer must
+  // not remove the failed wait's log and snapshots from the diagnostics artifact.
   let containerLogDirectory = join(clusters[0].directory, `container-logs-${randomUUID()}`);
   try {
     await mkdir(containerLogDirectory, { mode: 0o700 });
   } catch {
     containerLogDirectory = undefined;
   }
+  const watches = [];
+  try {
+    for (const cluster of clusters) {
+      const kubectl = cluster.kubectl ?? process.env.OCC_KUBECTL_BIN ?? "kubectl";
+      const scope = ["--kubeconfig", cluster.kubeconfig, "--context", cluster.context];
+      const capture = randomUUID();
+      const paths = {
+        events: join(cluster.directory, `agent-activity-${capture}-events.ndjson`),
+        pods: join(cluster.directory, `agent-activity-${capture}-pods.ndjson`),
+      };
+      const children = [];
+      // Register ownership before opening either stream so partial startup is cleaned up.
+      watches.push({ cluster, paths, children });
+      for (const [kind, path] of Object.entries(paths)) {
+        const query =
+          kind === "pods"
+            ? `/api/v1/pods?watch=true&timeoutSeconds=${WATCH_SECONDS}&labelSelector=${encodeURIComponent(AGENT_POD_SELECTOR)}`
+            : `/api/v1/events?watch=true&timeoutSeconds=${WATCH_SECONDS}`;
+        const initialQuery = `${query}&sendInitialEvents=true&allowWatchBookmarks=true&resourceVersionMatch=NotOlderThan`;
+        // Raw streams stay private; only their projection is published.
+        const output = await open(path, "w", 0o600);
+        let child;
+        try {
+          child = spawn(kubectl, [...scope, "get", "--raw", initialQuery], {
+            stdio: ["ignore", output.fd, "ignore"],
+          });
+          child.on("error", () => {});
+          children.push(child);
+        } finally {
+          await output.close();
+        }
+        // A spawned kubectl can still be connecting while a short-lived Pod disappears.
+        await waitForInitialWatch(child, path);
+      }
+    }
+  } catch {
+    await Promise.all(watches.flatMap(({ children }) => children).map(stopWatch));
+    await Promise.all(
+      watches.flatMap(({ paths }) => Object.values(paths)).map((path) => rm(path, { force: true })),
+    );
+    watches.length = 0;
+    console.error(`[run:${lane}] Agent namespace activity unavailable for ${file}`);
+  }
   return {
     env:
       containerLogDirectory === undefined

@@ -2282,6 +2282,7 @@ test("run keeps bounded Agent namespace activity from passing k3d files, alone a
       `#!${process.execPath}`,
       "const query = process.argv.at(-1);",
       `const lines = query.startsWith("/api/v1/pods?") ? ${JSON.stringify(pods)} : ${JSON.stringify(events)};`,
+      'process.stdout.write(JSON.stringify({ type: "BOOKMARK", object: { metadata: { annotations: { "k8s.io/initial-events-end": "true" } } } }) + "\\n");',
       "for (const line of lines) process.stdout.write(JSON.stringify(line) + '\\n');",
       'process.stdout.write(\'{"type":"MODIFIED","object":\');',
       // A live watch runs until stopped; a stranded one exits on its own after a minute.
@@ -2356,6 +2357,103 @@ test("run keeps bounded Agent namespace activity from passing k3d files, alone a
   // Raw watch streams hold full Pod specs; only the projection survives.
   assert.deepEqual(await readdir(clusterDirectory), []);
 
+  // A normal file can create and delete its namespace before kubectl has connected.
+  // The provider starts empty; a watch opened after that lifecycle cannot recover it.
+  const originalKubectl = await readFile(kubectl, "utf8");
+  const agentFile = join(root, "tests/integration/agent.test.mjs");
+  const originalAgentFile = await readFile(agentFile, "utf8");
+  const lifecycle = join(root, "short-lived-activity");
+  await writeFile(
+    kubectl,
+    [
+      `#!${process.execPath}`,
+      'import { existsSync } from "node:fs";',
+      'import { setTimeout as delay } from "node:timers/promises";',
+      "const query = process.argv.at(-1);",
+      `const lines = query.startsWith("/api/v1/pods?") ? ${JSON.stringify(pods)} : ${JSON.stringify(events)};`,
+      "await delay(800);",
+      `const alreadyGone = existsSync(${JSON.stringify(lifecycle)});`,
+      'process.stdout.write(JSON.stringify({ type: "BOOKMARK", object: { metadata: { annotations: { "k8s.io/initial-events-end": "true" } } } }) + "\\n");',
+      "if (!alreadyGone) {",
+      `  while (!existsSync(${JSON.stringify(lifecycle)})) await delay(10);`,
+      "  for (const line of lines) process.stdout.write(JSON.stringify(line) + '\\n');",
+      "}",
+      "setTimeout(() => {}, 60_000);",
+      "",
+    ].join("\n"),
+  );
+  await writeFile(
+    agentFile,
+    [
+      'import test from "node:test";',
+      'import { writeFile } from "node:fs/promises";',
+      'import { setTimeout as delay } from "node:timers/promises";',
+      'test("agent file passes", async () => {',
+      `  await writeFile(${JSON.stringify(lifecycle)}, "created and deleted");`,
+      "  await delay(100);",
+      "});",
+      "",
+    ].join("\n"),
+  );
+  const fastState = join(root, "state/k3d-fast.json");
+  await writeJson(fastState, JSON.parse(await readFile(statePath, "utf8")));
+  const fast = run(
+    root,
+    [
+      "run",
+      "k3d-lane",
+      "--manifest",
+      join(root, "scripts/ci/suites.json"),
+      "--root",
+      root,
+      "--state",
+      fastState,
+      "--results",
+      join(root, "results/k3d-fast.json"),
+    ],
+    { OCC_KUBECTL_BIN: kubectl },
+  );
+  assert.equal(fast.status, 0, fast.stderr);
+  const fastReport = JSON.parse(await readFile(`${fastState}.diagnostics.json`, "utf8"));
+  assert.deepEqual(fastReport.agentNamespaces[0].namespaces, ["occ-agent-a"]);
+  assert.deepEqual(
+    fastReport.agentNamespaces[0].pods.map(({ watch }) => watch),
+    ["ADDED", "MODIFIED", "DELETED"],
+  );
+  assert.deepEqual(await readdir(clusterDirectory), []);
+  await writeFile(kubectl, originalKubectl);
+  await writeFile(agentFile, originalAgentFile);
+  await rm(lifecycle);
+
+  // Optional diagnostics must not strand a file when an alive watch never synchronizes.
+  await writeFile(kubectl, `#!${process.execPath}\nsetTimeout(() => {}, 60_000);\n`);
+  const stalledState = join(root, "state/k3d-stalled.json");
+  await writeJson(stalledState, JSON.parse(await readFile(statePath, "utf8")));
+  const stalled = run(
+    root,
+    [
+      "run",
+      "k3d-lane",
+      "--manifest",
+      join(root, "scripts/ci/suites.json"),
+      "--root",
+      root,
+      "--state",
+      stalledState,
+      "--results",
+      join(root, "results/k3d-stalled.json"),
+    ],
+    { OCC_KUBECTL_BIN: kubectl },
+  );
+  assert.equal(stalled.status, 0, stalled.stderr);
+  assert.match(stalled.stderr, /Agent namespace activity unavailable/);
+  assert.deepEqual(await readdir(clusterDirectory), []);
+  assert.equal(
+    JSON.parse(await readFile(join(root, "results/k3d-stalled.json"), "utf8")).counts.passed,
+    1,
+  );
+  await writeFile(kubectl, originalKubectl);
+
   // Two files sharing the runner under fileConcurrency watch the same cluster at once. Each
   // keeps its own watch streams, so neither truncates nor deletes the other's capture.
   await writeFile(
@@ -3363,6 +3461,9 @@ test("run publishes a failed wait's followed container log, redacted, beside Age
       '} else if (args.includes("get") && args.includes("events")) {',
       `  process.stdout.write(JSON.stringify({ items: [${JSON.stringify(event)}] }));`,
       "} else {",
+      '  if (process.env.CI_RUNNER_STALLED_WATCH !== "1") {',
+      '  process.stdout.write(JSON.stringify({ type: "BOOKMARK", object: { metadata: { annotations: { "k8s.io/initial-events-end": "true" } } } }) + "\\n");',
+      "  }",
       "  setTimeout(() => {}, 60_000);",
       "}",
       "",
@@ -3488,4 +3589,47 @@ test("run publishes a failed wait's followed container log, redacted, beside Age
     (await readdir(clusterDirectory)).filter((name) => name.startsWith("container-logs-")),
     [],
   );
+
+  // A live observer that never sends its bookmark must not disable the independent
+  // failed-wait log, snapshots or the file's original failure result.
+  const stalledState = join(root, "state/k3d-stalled-wait.json");
+  const stalledResults = join(root, "results/k3d-stalled-wait.json");
+  await writeJson(stalledState, JSON.parse(await readFile(statePath, "utf8")));
+  const stalled = run(
+    root,
+    [
+      "run",
+      "k3d-lane",
+      "--manifest",
+      join(root, "scripts/ci/suites.json"),
+      "--root",
+      root,
+      "--state",
+      stalledState,
+      "--results",
+      stalledResults,
+    ],
+    {
+      OCC_KUBECTL_BIN: kubectl,
+      PATH: `${bin}:${process.env.PATH}`,
+      CI_RUNNER_STALLED_WATCH: "1",
+    },
+  );
+  assert.equal(stalled.status, 1, stalled.stderr);
+  assert.match(stalled.stderr, /Agent namespace activity unavailable/);
+  const stalledSummary = JSON.parse(await readFile(stalledResults, "utf8"));
+  assert.equal(stalledSummary.counts.failed, 1);
+  assert.equal(stalledSummary.counts.passed, 1);
+  const stalledText = await readFile(`${stalledState}.diagnostics.json`, "utf8");
+  const stalledReport = JSON.parse(stalledText);
+  assert.equal(stalledReport.agentNamespaces, undefined);
+  assert.equal(stalledReport.containerLogs?.length, 1);
+  const [stalledLog] = stalledReport.containerLogs;
+  assert.equal(stalledLog.reason, log.reason);
+  assert.deepEqual(stalledLog.lines, log.lines);
+  assert.deepEqual(stalledLog.snapshots[0].pods, snapshot.pods);
+  assert.deepEqual(stalledLog.snapshots[0].events, snapshot.events);
+  assert.equal(stalledLog.snapshots[1].unavailable, true);
+  assert.doesNotMatch(stalledText, /do-not-publish|secretauthvalue|childonlysecret/);
+  assert.deepEqual(await readdir(clusterDirectory), []);
 });