Honor checkpoint tombstones in bounded node liveness (#894)

CHANGELOG.md

@@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
+- Bounded checkpoint-tail node reads honor checkpoint tombstones for late
+  arriving adds and refuse unwitnessed adds instead of resurrecting a node
+  already removed by the checkpoint (#894).
+
 - Atomic Intent arrays reject descriptors exceeding 16 MiB before allocating
   their complete canonical string or UTF-8 buffer. Exact byte accounting covers
   aggregate members, escaping, Unicode, nested properties and inline binary

src/domain/services/optic/CheckpointTailFactReducer.ts

@@ -1,3 +1,4 @@
+import type { CheckpointNodeLifecycleRecord } from './CheckpointShardFactReader.ts';
 import { LWWRegister } from '../../crdt/LWW.ts';
 import QueryError from '../../errors/QueryError.ts';
 import NodeAdd from '../../types/ops/NodeAdd.ts';
@@ -95,16 +96,22 @@ export default class CheckpointTailFactReducer {
     return Object.freeze([...nodeIds].sort());
   }
 
-  reduceNodeLiveness(
-    baseAlive: boolean,
-    tailEntries: readonly CheckpointTailPatchEntry[],
-    nodeId: string,
-  ): boolean {
-    let alive = baseAlive;
-    for (const entry of tailEntries) {
-      alive = this._reduceNodeLivenessEntry(alive, entry, nodeId);
+  reduceNodeLiveness(options: {
+    readonly baseAlive: boolean;
+    readonly lifecycle: CheckpointNodeLifecycleRecord;
+    readonly tailEntries: readonly CheckpointTailPatchEntry[];
+    readonly nodeId: string;
+  }): boolean {
+    this._assertNoNodeRemoves(options.tailEntries, options.nodeId);
+    const tail = CheckpointTailNodeScan.scan(options.tailEntries, options.nodeId);
+    if (options.lifecycle.kind === 'unwitnessed') {
+      if (tail.hasAdd()) {
+        throwNoBoundedBasis(this._graphName, 'tail-node-add-needs-checkpoint-lifecycle-witnesses');
+      }
+      return options.baseAlive;
     }
-    return alive;
+    // Tail removes were refused above, so this scan cannot be undecided.
+    return tail.livenessAfter(options.baseAlive, options.lifecycle.floatingTombstones) === 'alive';
   }
 
   reduceProperty(options: {
@@ -185,30 +192,12 @@ export default class CheckpointTailFactReducer {
     }
   }
 
-  private _reduceNodeLivenessEntry(
-    currentAlive: boolean,
-    entry: CheckpointTailPatchEntry,
-    nodeId: string,
-  ): boolean {
-    let alive = currentAlive;
-    for (const rawOp of entry.patch.ops) {
-      alive = this._reduceNodeLivenessOp(alive, normalizeRawOp(rawOp), nodeId);
-    }
-    return alive;
-  }
-
-  private _reduceNodeLivenessOp(
-    currentAlive: boolean,
-    op: NormalizedTailOperation,
-    nodeId: string,
-  ): boolean {
-    if (isTargetNodeAdd(op, nodeId)) {
-      return true;
-    }
-    if (isTargetNodeRemove(op, nodeId)) {
-      throwNoBoundedBasis(this._graphName, 'tail-node-remove-needs-raw-liveness-witnesses');
+  private _assertNoNodeRemoves(entries: readonly CheckpointTailPatchEntry[], nodeId: string): void {
+    for (const entry of entries) {
+      if (entry.patch.ops.some((op) => isTargetNodeRemove(normalizeRawOp(op), nodeId))) {
+        throwNoBoundedBasis(this._graphName, 'tail-node-remove-needs-raw-liveness-witnesses');
+      }
     }
-    return currentAlive;
   }
 
   private _tailPropertyRegister(options: {

src/domain/services/optic/CheckpointTailWitnessLocator.ts

@@ -174,14 +174,18 @@ export default class CheckpointTailWitnessLocator {
     const basis = await this._basisLoader.load();
     const baseAlive = await this._shardReader.readNodeAlive(basis, nodeId);
     const tail = await this._scanTailForNode(basis, nodeId);
-    const alive = this._factReducer.reduceNodeLiveness(baseAlive, tail.entries, nodeId);
+    const lifecycle = await this._shardReader.readNodeLifecycle(basis, nodeId);
+    const alive = this._factReducer.reduceNodeLiveness({ baseAlive, lifecycle, tailEntries: tail.entries, nodeId });
     return new NodeOpticReadResult({
       nodeId,
       alive,
       readIdentity: this._readIdentityBuilder.nodeLiveness({
         basis,
         nodeId,
-        checkpointIndexShards: this._shardReader.nodeLivenessShardIdentities(basis, nodeId),
+        checkpointIndexShards: [
+          ...this._shardReader.nodeLivenessShardIdentities(basis, nodeId),
+          ...this._shardReader.nodeLifecycleShardIdentities(basis, nodeId),
+        ],
         tailWitnesses: tail.witnesses,
       }),
     });

test/helpers/CheckpointTailLifecycleHarness.ts

@@ -26,6 +26,7 @@ import CheckpointTailOpticSource, {
 import CheckpointTailFactReducer from '../../src/domain/services/optic/CheckpointTailFactReducer.ts';
 import CheckpointTailWitnessLocator from '../../src/domain/services/optic/CheckpointTailWitnessLocator.ts';
 import Optic from '../../src/domain/services/optic/Optic.ts';
+import type NodeOpticReadResult from '../../src/domain/services/optic/NodeOpticReadResult.ts';
 import OpticAperturePosture from '../../src/domain/services/optic/OpticAperturePosture.ts';
 import OpticBasisPosture from '../../src/domain/services/optic/OpticBasisPosture.ts';
 import OpticCoordinatePosture from '../../src/domain/services/optic/OpticCoordinatePosture.ts';
@@ -129,10 +130,29 @@ export async function tailRead(
     const result = await locator.readNodeProperty(propertyOptic());
     return { kind: 'value', value: result.value ?? null };
   } catch (error) {
+    if (!(error instanceof QueryError)) { throw error; }
     return { kind: 'refused', reason: refusalReason(error) };
   }
 }
 
+/** Reads node membership through the production locator over checkpoint-writer roots. */
+export async function tailNodeRead(
+  scenario: HarnessScenario,
+  options: Readonly<{ recordsAbsent?: boolean; damage?: HarnessRootDamage }> = {},
+): Promise<NodeOpticReadResult> {
+  const indexStore = new MockIndexStorage();
+  const written = await writeBasis(replay(stepsOf(scenario.checkpoint)), indexStore, options.recordsAbsent ?? false);
+  const roots = options.damage === undefined ? written : await damageRoot(written, indexStore, options.damage);
+  const locator = new CheckpointTailWitnessLocator({ source: new HarnessSource({ indexStore, scenario, roots }) });
+  return await locator.readNode(Optic.node({
+    coordinatePosture: OpticCoordinatePosture.capturedCoordinate(),
+    aperturePosture: OpticAperturePosture.defaultFullRead(),
+    basisPosture: OpticBasisPosture.checkpointTailBasisVerified(),
+    evidencePosture: ContinuumEvidencePosture.translatedGitWarpEvidence(),
+    nodeId: HARNESS_NODE,
+  }));
+}
+
 /**
  * The reducer's answer for every order of the tail patches, given what the
  * written basis carries: the checkpoint's lifecycle records, the register
@@ -254,15 +274,9 @@ function propertyOptic(): Optic {
   });
 }
 
-function refusalReason(error: unknown): string {
-  if (!(error instanceof Error)) {
-    throw error;
-  }
-  const context: unknown = Reflect.get(error, 'context');
-  const cause = typeof context === 'object' && context !== null ? Reflect.get(context, 'cause') : undefined;
-  if (typeof cause !== 'string') {
-    throw error;
-  }
+function refusalReason(error: QueryError): string {
+  const cause = error.context['cause'];
+  if (typeof cause !== 'string') { throw error; }
   return cause;
 }
 

test/unit/domain/services/optic/CheckpointTailWitnessLocator.nodeLiveness.test.ts

@@ -0,0 +1,53 @@
+import { describe, expect, it } from 'vitest';
+import { Dot, encodeDot } from '../../../../../src/domain/crdt/Dot.ts';
+import NodeAdd from '../../../../../src/domain/types/ops/NodeAdd.ts';
+import NodeRemove from '../../../../../src/domain/types/ops/NodeRemove.ts';
+import { NODE_LIFECYCLE_RECEIPT_PATH } from '../../../../../src/domain/services/index/NodeLifecycleShardReader.ts';
+import {
+  HARNESS_NODE, replay, stepsOf, tailNodeRead, type HarnessScenario,
+} from '../../../../helpers/CheckpointTailLifecycleHarness.ts';
+
+const removedDot = Dot.create('Y', 1);
+const removedAdd = { writer: 'Y', lamport: 3, sha: 'ffff0003', ops: [new NodeAdd(HARNESS_NODE, removedDot)] };
+const remove = { writer: 'X', lamport: 5, sha: 'dddd0005', ops: [new NodeRemove(HARNESS_NODE, [encodeDot(removedDot)])] };
+const independentAdd = { writer: 'A', lamport: 1, sha: 'aaaa0001', ops: [new NodeAdd(HARNESS_NODE, Dot.create('A', 1))] };
+const removedScenario: HarnessScenario = { checkpoint: [remove], tail: [removedAdd] };
+
+describe('checkpoint-tail node liveness with floating removed dots (#894)', () => {
+  it.each([
+    { name: 'only the removed dot exists', scenario: removedScenario, expected: false },
+    { name: 'an independent checkpoint add survives', scenario: { checkpoint: [independentAdd, remove], tail: [removedAdd] }, expected: true },
+    { name: 'an independent tail add survives', scenario: { checkpoint: [remove], tail: [removedAdd, independentAdd] }, expected: true },
+  ])('agrees with full membership when $name', async ({ scenario, expected }) => {
+    const all = [...scenario.checkpoint, ...scenario.tail];
+    for (const patches of [all, [...all].reverse()]) {
+      expect(replay(stepsOf(patches)).nodeAlive.contains(HARNESS_NODE)).toBe(expected);
+    }
+    for (const tail of [scenario.tail, [...scenario.tail].reverse()]) {
+      expect((await tailNodeRead({ checkpoint: scenario.checkpoint, tail })).alive).toBe(expected);
+    }
+  });
+
+  it('includes the tombstone receipt in read identity', async () => {
+    const read = await tailNodeRead(removedScenario);
+    expect(read.readIdentity.checkpointIndexShards.map((shard) => shard.path)).toContain(NODE_LIFECYCLE_RECEIPT_PATH);
+  });
+
+  it('reads unchanged checkpoint membership without legacy lifecycle records', async () => {
+    const read = await tailNodeRead({ checkpoint: [independentAdd], tail: [] }, { recordsAbsent: true });
+    expect(read.alive).toBe(true);
+  });
+
+  it('refuses an invalid lifecycle receipt instead of assuming no tombstones', async () => {
+    await expect(tailNodeRead(removedScenario, { damage: 'receipt-schema-version' })).rejects.toMatchObject({
+      code: 'E_OPTIC_NO_BOUNDED_BASIS', context: { cause: 'checkpoint-shard-invalid' },
+    });
+  });
+
+  it('refuses a tail add when the checkpoint has no lifecycle receipt', async () => {
+    await expect(tailNodeRead(removedScenario, { recordsAbsent: true })).rejects.toMatchObject({
+      code: 'E_OPTIC_NO_BOUNDED_BASIS',
+      context: { cause: 'tail-node-add-needs-checkpoint-lifecycle-witnesses' },
+    });
+  });
+});