Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
66 changes: 62 additions & 4 deletions tests/droid/runs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -664,6 +664,58 @@ describe('droid runs', () => {
expect(reconcilePayload.data.summary).toBe('Command completed with exit code 0.');
});

it('uses live run room activity to guard reconcile', async () => {
vi.useFakeTimers();
vi.setSystemTime(new Date('2026-05-14T12:00:00.000Z'));
const rooms = new FakeRunRoomNamespace({ createdAt: () => new Date().toISOString() });
const app = createApp({
async execute(input) {
await input.recordEvent({ type: 'agent_process_start', command: input.command });
return new Promise(() => undefined);
},
async reconcile() {
return { stdout: 'reconciled', stderr: '', exitCode: 0, success: true };
},
});
const env = createEnv({ DROID_RUN_ROOMS: rooms as unknown as DurableObjectNamespace });

try {
const createResponse = await app.request(
'/v0/runs',
{
method: 'POST',
body: JSON.stringify({ command: 'sleep forever' }),
headers: {
Authorization: 'Bearer test-token',
'Content-Type': 'application/json',
},
},
env
);
const createPayload = (await createResponse.json()) as { data: { id: string } };

const reconcileResponse = await app.request(
`/v0/runs/${createPayload.data.id}/reconcile`,
{
method: 'POST',
body: JSON.stringify({ wait_for_completion: true }),
headers: {
Authorization: 'Bearer test-token',
'Content-Type': 'application/json',
},
},
env
);

expect(reconcileResponse.status).toBe(409);
await expect(reconcileResponse.json()).resolves.toMatchObject({
latest_event_source: 'run_room',
});
} finally {
vi.useRealTimers();
}
});

it('fails and cancels runs that exceed the hard Droid timeout', async () => {
vi.useFakeTimers();
const cancelCalls: string[] = [];
Expand Down Expand Up @@ -780,10 +832,12 @@ class FakeD1 {
class FakeRunRoomNamespace {
private rooms = new Map<string, FakeRunRoom>();

constructor(private options: { createdAt?: (index: number) => string } = {}) {}

getByName(name: string) {
let room = this.rooms.get(name);
if (!room) {
room = new FakeRunRoom(name);
room = new FakeRunRoom(name, this.options);
this.rooms.set(name, room);
}
return room;
Expand All @@ -793,14 +847,18 @@ class FakeRunRoomNamespace {
class FakeRunRoom {
private events: Array<Record<string, unknown>> = [];

constructor(private runId: string) {}
constructor(
private runId: string,
private options: { createdAt?: (index: number) => string } = {}
) {}

async recordEvent(input: { runId: string; event: { type: string } & Record<string, unknown> }) {
const index = this.events.length;
const event = {
id: `event-${this.events.length}`,
id: `event-${index}`,
run_id: input.runId,
...input.event,
created_at: `2026-05-11T00:00:0${this.events.length}.000Z`,
created_at: this.options.createdAt?.(index) ?? `2026-05-11T00:00:0${index}.000Z`,
};
this.events.push(event);
return event;
Expand Down
38 changes: 36 additions & 2 deletions workers/droid/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ export function createApp(executor: RunExecutor) {
}

if (!executor.reconcile) return c.json({ error: 'Run reconciliation is not supported' }, 501);
const latestEvent = await getLatestRunEvent(c.env, run.id);
const latestEvent = await getLatestRunActivity(c.env, run.id);
if (
!incoming?.force &&
latestEvent &&
Expand All @@ -376,6 +376,7 @@ export function createApp(executor: RunExecutor) {
error:
'Run still appears active; reconcile is only allowed after 6 minutes of no events unless force is true.',
latest_event_at: latestEvent.created_at,
latest_event_source: latestEvent.source,
},
409
);
Expand Down Expand Up @@ -425,12 +426,45 @@ function scheduleBackground(
}
}

async function getLatestRunActivity(
env: Env,
runId: string
): Promise<{ created_at: string; source: 'd1' | 'run_room' } | null> {
const candidates: Array<{ created_at: string; source: 'd1' | 'run_room'; parsed: number }> = [];
const latestEvent = await getLatestRunEvent(env, runId);
const latestEventTime = latestEvent ? parseRunTimestamp(latestEvent.created_at) : Number.NaN;
if (latestEvent && Number.isFinite(latestEventTime)) {
candidates.push({ created_at: latestEvent.created_at, source: 'd1', parsed: latestEventTime });
}

const roomStatus = await getRunRoomStatus(env, runId);

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Tolerate run-room status failures during reconcile

When DROID_RUN_ROOMS is configured but the Durable Object lookup/status call fails transiently, this new awaited call makes /v0/runs/:id/reconcile hit the app-level 500 handler before it can fall back to the D1 event history. Run-room writes are already treated as best-effort in createRunEvent, so reconcile should similarly catch/log getRunRoomStatus failures and continue using the D1 candidate; otherwise operators can lose the recovery path for stale running rows exactly when the live room is unavailable.

Useful? React with 👍 / 👎.

const roomEventTime = roomStatus?.last_event_at
? parseRunTimestamp(roomStatus.last_event_at)
: Number.NaN;
if (roomStatus?.last_event_at && Number.isFinite(roomEventTime)) {
candidates.push({
created_at: roomStatus.last_event_at,
source: 'run_room',
parsed: roomEventTime,
});
}

candidates.sort((left, right) => right.parsed - left.parsed);
const latest = candidates[0];
return latest ? { created_at: latest.created_at, source: latest.source } : null;
}

function isStaleEvent(createdAt: string, thresholdMs: number): boolean {
const parsed = Date.parse(`${createdAt.replace(' ', 'T')}Z`);
const parsed = parseRunTimestamp(createdAt);
if (!Number.isFinite(parsed)) return false;
return Date.now() - parsed >= thresholdMs;
}

function parseRunTimestamp(value: string): number {
if (value.includes('T')) return Date.parse(value);
return Date.parse(`${value.replace(' ', 'T')}Z`);
}

async function createRunEvent(env: Env, runId: string, input: RunEventInput): Promise<void> {
await createRunEventInDb(env, runId, input);
try {
Expand Down
Loading