fix(checkpoints): publish durable refs and preserve captured state

.agents/upstream-review.md

@@ -181,3 +181,4 @@ acknowledged renderer navigation. No mobile push or background daemon delivery.
 The review cursor stays unchanged; unrelated sources in this range remain unclassified.
 
 | Antigravity non-spawning health and owned runtime temp / `6d1d549441be84f19696ab59ed7e2fbf305280d4` | `8c18b5bb21a5349fbadba8c77456ea34c41b7339` (#12008) | Adopted with Pylon adaptation: installation-only probes and per-process scoped temp directories. Intentionally exclude startup deletion of the shared temp root because other drivers/servers may still own its contents; failed cleanup or abrupt termination may leave directories. No system-temp cleanup. | Cycle #585; focused driver/environment regressions and server checks in implementation PR. |
+| Durable checkpoint writes and monotonic projection / `6d1d549441be84f19696ab59ed7e2fbf305280d4` | `8130a9f13a1ef506f4393423ded21d71f3c9b5e4` (#10944), `d42a3bd2ae3e698141b26f6426aa28d8886e1754` (#8432) | Adopted: fsync object/ref writes before publication; SQL preserves ready/error checkpoints against later missing placeholders. Pylon temporary index, identity, interrupted/running turn handling and rollback leases retained. Existing corrupt refs are not repaired; older Git may ignore unsupported fsync settings. | Cycle #585; focused Git capture, real CheckpointStore and projection tests in implementation PR. |

apps/server/src/orchestration/Layers/ProjectionPipeline.test.ts

@@ -4667,6 +4667,130 @@ it.layer(BaseTestLayer)("OrchestrationProjectionPipeline", (it) => {
       ]);
     }),
   );
+
+  it.effect("does not let a later missing placeholder clobber a ready checkpoint", () =>
+    Effect.gen(function* () {
+      const projectionPipeline = yield* OrchestrationProjectionPipeline;
+      const eventStore = yield* OrchestrationEventStore;
+      const sql = yield* SqlClient.SqlClient;
+      const appendAndProject = (event: Parameters<typeof eventStore.append>[0]) =>
+        eventStore
+          .append(event)
+          .pipe(Effect.flatMap((savedEvent) => projectionPipeline.projectEvent(savedEvent)));
+
+      yield* appendAndProject({
+        type: "project.created",
+        eventId: EventId.make("evt-checkpoint-guard-1"),
+        aggregateKind: "project",
+        aggregateId: ProjectId.make("project-checkpoint-guard"),
+        occurredAt: "2026-02-26T14:00:00.000Z",
+        commandId: CommandId.make("cmd-checkpoint-guard-1"),
+        causationEventId: null,
+        correlationId: CorrelationId.make("cmd-checkpoint-guard-1"),
+        metadata: {},
+        payload: {
+          projectId: ProjectId.make("project-checkpoint-guard"),
+          title: "Project Checkpoint Guard",
+          workspaceRoot: "/tmp/project-checkpoint-guard",
+          defaultModelSelection: null,
+          scripts: [],
+          createdAt: "2026-02-26T14:00:00.000Z",
+          updatedAt: "2026-02-26T14:00:00.000Z",
+        },
+      });
+
+      yield* appendAndProject({
+        type: "thread.created",
+        eventId: EventId.make("evt-checkpoint-guard-2"),
+        aggregateKind: "thread",
+        aggregateId: ThreadId.make("thread-checkpoint-guard"),
+        occurredAt: "2026-02-26T14:00:01.000Z",
+        commandId: CommandId.make("cmd-checkpoint-guard-2"),
+        causationEventId: null,
+        correlationId: CorrelationId.make("cmd-checkpoint-guard-2"),
+        metadata: {},
+        payload: {
+          threadId: ThreadId.make("thread-checkpoint-guard"),
+          projectId: ProjectId.make("project-checkpoint-guard"),
+          title: "Thread Checkpoint Guard",
+          modelSelection: {
+            instanceId: ProviderInstanceId.make("codex"),
+            model: "gpt-5-codex",
+          },
+          runtimeMode: "full-access",
+          interactionMode: "default",
+          branch: null,
+          worktreePath: null,
+          createdAt: "2026-02-26T14:00:01.000Z",
+          updatedAt: "2026-02-26T14:00:01.000Z",
+        },
+      });
+
+      yield* appendAndProject({
+        type: "thread.turn-diff-completed",
+        eventId: EventId.make("evt-checkpoint-guard-3"),
+        aggregateKind: "thread",
+        aggregateId: ThreadId.make("thread-checkpoint-guard"),
+        occurredAt: "2026-02-26T14:00:02.000Z",
+        commandId: CommandId.make("cmd-checkpoint-guard-3"),
+        causationEventId: null,
+        correlationId: CorrelationId.make("cmd-checkpoint-guard-3"),
+        metadata: {},
+        payload: {
+          threadId: ThreadId.make("thread-checkpoint-guard"),
+          turnId: TurnId.make("turn-ready"),
+          checkpointTurnCount: 1,
+          checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-checkpoint-guard/turn/1"),
+          status: "ready",
+          files: [],
+          assistantMessageId: MessageId.make("assistant-ready"),
+          completedAt: "2026-02-26T14:00:02.000Z",
+        },
+      });
+
+      yield* appendAndProject({
+        type: "thread.turn-diff-completed",
+        eventId: EventId.make("evt-checkpoint-guard-4"),
+        aggregateKind: "thread",
+        aggregateId: ThreadId.make("thread-checkpoint-guard"),
+        occurredAt: "2026-02-26T14:00:03.000Z",
+        commandId: CommandId.make("cmd-checkpoint-guard-4"),
+        causationEventId: null,
+        correlationId: CorrelationId.make("cmd-checkpoint-guard-4"),
+        metadata: {},
+        payload: {
+          threadId: ThreadId.make("thread-checkpoint-guard"),
+          turnId: TurnId.make("turn-ready"),
+          checkpointTurnCount: 1,
+          checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread-checkpoint-guard/turn/1"),
+          status: "missing",
+          files: [],
+          assistantMessageId: MessageId.make("assistant-ready"),
+          completedAt: "2026-02-26T14:00:03.000Z",
+        },
+      });
+
+      const turnRows = yield* sql<{
+        readonly turnId: string;
+        readonly checkpointStatus: string | null;
+        readonly checkpointRef: string | null;
+      }>`
+        SELECT
+          turn_id AS "turnId",
+          checkpoint_status AS "checkpointStatus",
+          checkpoint_ref AS "checkpointRef"
+        FROM projection_turns
+        WHERE thread_id = 'thread-checkpoint-guard'
+      `;
+      assert.deepEqual(turnRows, [
+        {
+          turnId: "turn-ready",
+          checkpointStatus: "ready",
+          checkpointRef: "refs/t3/checkpoints/thread-checkpoint-guard/turn/1",
+        },
+      ]);
+    }),
+  );
 });
 
 it.layer(makeProjectionPipelinePrefixedTestLayer("t3-pending-turn-terminal-test-"))(

apps/server/src/orchestration/Layers/ProjectionPipeline.ts

@@ -1786,6 +1786,18 @@ const makeOrchestrationProjectionPipeline = Effect.fn("makeOrchestrationProjecti
             threadId: event.payload.threadId,
             turnId: event.payload.turnId,
           });
+          // Do not let a placeholder (status "missing") overwrite a checkpoint
+          // that has already been captured with a real git ref. In-memory
+          // projectEvent already refuses this. SQL did not, so thread detail
+          // and diffs could show missing after a ready capture.
+          if (
+            Option.isSome(existingTurn) &&
+            existingTurn.value.checkpointStatus !== null &&
+            existingTurn.value.checkpointStatus !== "missing" &&
+            event.payload.status === "missing"
+          ) {
+            return;
+          }
           const nextState = event.payload.status === "error" ? "error" : "completed";
           yield* projectionTurnRepository.clearCheckpointTurnConflict({
             threadId: event.payload.threadId,

apps/server/src/vcs/GitVcsDriver.test.ts

@@ -176,3 +176,67 @@ it.effect("GitVcsDriver forwards execute env to the VCS process", () => {
     ),
   );
 });
+
+it.effect("GitVcsDriver flushes checkpoint objects and refs to disk before publishing them", () => {
+  const observedArgs: ReadonlyArray<string>[] = [];
+
+  return Effect.gen(function* () {
+    const driver = yield* GitVcsDriver.makeVcsDriverShape();
+
+    yield* driver.checkpoints.captureCheckpoint({
+      cwd: "/repo",
+      checkpointRef: CheckpointRef.make("refs/t3/checkpoints/thread/turn/1"),
+    });
+
+    const writeCommands = ["add", "write-tree", "commit-tree", "update-ref"];
+    const writes = observedArgs.filter((args) =>
+      writeCommands.some((command) => args.includes(command)),
+    );
+    assert.strictEqual(writes.length, 4);
+    for (const args of writes) {
+      const command = args.findIndex((arg) => writeCommands.includes(arg));
+      for (const setting of ["core.fsync=objects,reference", "core.fsyncMethod=fsync"]) {
+        const index = args.indexOf(setting);
+        assert.strictEqual(args[index - 1], "-c", args.join(" "));
+        assert.isBelow(index, command);
+      }
+    }
+    assert.deepStrictEqual(observedArgs.at(-1), [
+      "-C",
+      "/repo",
+      "-c",
+      "core.fsync=objects,reference",
+      "-c",
+      "core.fsyncMethod=fsync",
+      "update-ref",
+      "refs/t3/checkpoints/thread/turn/1",
+      "commit0000",
+    ]);
+  }).pipe(
+    Effect.provide(
+      Layer.mergeAll(
+        NodeServices.layer,
+        Layer.mock(VcsProcess.VcsProcess)({
+          run: (input) =>
+            Effect.sync(() => {
+              observedArgs.push(input.args);
+              const stdout = input.args.includes("write-tree")
+                ? "tree0000\n"
+                : input.args.includes("commit-tree")
+                  ? "commit0000\n"
+                  : input.args.includes("--git-common-dir")
+                    ? ".git\n"
+                    : "";
+              return {
+                exitCode: ChildProcessSpawner.ExitCode(0),
+                stdout,
+                stderr: "",
+                stdoutTruncated: false,
+                stderrTruncated: false,
+              };
+            }),
+        }),
+      ),
+    ),
+  );
+});

apps/server/src/vcs/GitVcsDriver.ts

@@ -717,6 +717,17 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
       return path.isAbsolute(gitCommonDir) ? gitCommonDir : path.resolve(cwd, gitCommonDir);
     });
 
+  // Git renames loose objects and refs into place without fsync by default, so
+  // an unclean restart can leave 0-byte files under refs/t3/** that break every
+  // later fetch and push. Checkpoint writes flush before they are published;
+  // macOS defaults to writeout-only, which does not reach the disk either.
+  const durableWrite = [
+    "-c",
+    "core.fsync=objects,reference",
+    "-c",
+    "core.fsyncMethod=fsync",
+  ] as const;
+
   const checkpoints: VcsDriver.VcsCheckpointOps = {
     captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {
       const operation = "GitVcsDriver.checkpoints.captureCheckpoint";
@@ -752,14 +763,14 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
         yield* execute({
           operation,
           cwd: input.cwd,
-          args: ["add", "-A", "--", "."],
+          args: [...durableWrite, "add", "-A", "--", "."],
           env: commitEnv,
         });
 
         const writeTreeResult = yield* execute({
           operation,
           cwd: input.cwd,
-          args: ["write-tree"],
+          args: [...durableWrite, "write-tree"],
           env: commitEnv,
         });
         const treeOid = writeTreeResult.stdout.trim();
@@ -777,7 +788,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
         const commitTreeResult = yield* execute({
           operation,
           cwd: input.cwd,
-          args: ["commit-tree", treeOid, "-m", message],
+          args: [...durableWrite, "commit-tree", treeOid, "-m", message],
           env: commitEnv,
         });
         const commitOid = commitTreeResult.stdout.trim();
@@ -794,7 +805,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
         yield* execute({
           operation,
           cwd: input.cwd,
-          args: ["update-ref", input.checkpointRef, commitOid],
+          args: [...durableWrite, "update-ref", input.checkpointRef, commitOid],
         });
       }).pipe(Effect.ensuring(cleanupTempIndex));
     }),