Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,12 @@ import type {
ConnectOrSpawnRuntimeHostInput,
RuntimeHostConnection,
} from '@maka/runtime-host/client';
import { RuntimeHostOperationError, runHostHandoff } from '@maka/runtime-host/client';
import {
DEFAULT_ELECTION_DEADLINE_MS,
ELECTION_DEADLINE_MS_ENV_VAR,
RuntimeHostOperationError,
runHostHandoff,
} from '@maka/runtime-host/client';
import {
SESSION_CONTINUITY_SCHEMA_VERSION,
type ClientCapabilityCallFrame,
Expand All @@ -52,6 +57,7 @@ import { createAttachmentApprovalRegistry } from '../attachment-approval.js';
import {
createDesktopRuntimeHostCandidate as createCandidate,
formatLocalRuntimeHostProcessExitDiagnostic,
resolveDesktopCandidateReadyTimeoutMs,
startDesktopRuntimeHostCandidate,
type DesktopRuntimeHostCandidateControls,
type DesktopRuntimeHostCandidateDeps,
Expand Down Expand Up @@ -282,6 +288,22 @@ test('does not claim a blank local Host stderr tail was truncated', () => {
assert.doesNotMatch(diagnostic, /stderr:|stderr truncated/);
});

test('the Desktop candidate waits on the window the client elects on', () => {
assert.equal(
resolveDesktopCandidateReadyTimeoutMs(undefined, {}),
DEFAULT_ELECTION_DEADLINE_MS,
'with no caller value the Desktop wait must land on the client default, not its own number',
);
assert.equal(
resolveDesktopCandidateReadyTimeoutMs(undefined, {
[ELECTION_DEADLINE_MS_ENV_VAR]: '90000',
}),
90_000,
'the operator override has to widen the Desktop wait too, or it only widens the election',
);
assert.equal(resolveDesktopCandidateReadyTimeoutMs(5_000, {}), 5_000);
});

function createDesktopRuntimeHostCandidate(
connection: RuntimeHostConnection,
candidateDeps: DesktopRuntimeHostCandidateDeps,
Expand Down
18 changes: 17 additions & 1 deletion apps/desktop/src/main/runtime-host-desktop-candidate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,9 @@ import {
type RuntimeHostSshOperatorActivationInput,
connectOrSpawnRuntimeHost,
connectRuntimeHostProfile,
DEFAULT_ELECTION_DEADLINE_MS,
ELECTION_DEADLINE_MS_ENV_VAR,
electionDeadlineMsFromEnvironment,
type RuntimeHostPeerClient,
type RuntimeHostConnectionPhase,
type RuntimeHostSshInteraction,
Expand Down Expand Up @@ -492,6 +495,19 @@ function redactRuntimeHostStderr(stderr: string): string {
return redacted.replace(/\S+/gu, (token) => redactSecrets(token));
}

// Resolved from the client's own constant and env channel: a Desktop-only
// fallback let the readiness wait expire inside a still-valid election window.
export function resolveDesktopCandidateReadyTimeoutMs(
electionDeadlineMs: number | undefined,
environment: Readonly<Record<string, string | undefined>> = process.env,
): number {
return (
electionDeadlineMs ??
electionDeadlineMsFromEnvironment(environment[ELECTION_DEADLINE_MS_ENV_VAR]) ??
DEFAULT_ELECTION_DEADLINE_MS
);
}

async function startProfileDesktopRuntimeHostCandidate(
input: DesktopRuntimeHostCandidateStartInput,
profileTarget: NonNullable<DesktopRuntimeHostCandidateStartInput["profileTarget"]>,
Expand All @@ -511,7 +527,7 @@ async function startProfileDesktopRuntimeHostCandidate(
...(input.handshakeTimeoutMs === undefined
? {}
: { handshakeTimeoutMs: input.handshakeTimeoutMs }),
readyTimeoutMs: input.electionDeadlineMs ?? 45_000,
readyTimeoutMs: resolveDesktopCandidateReadyTimeoutMs(input.electionDeadlineMs),

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P3: the new value goes to the remote/WSL profile path, not the local election that #5843 is about. startProfileDesktopRuntimeHostCandidate is only reached when input.profileTarget is set (:383). It calls connectRuntimeHostProfile, which serves remote SSH/WebSocket profiles (host-profile.ts:592-596) and WSL environment profiles (wsl-environment.ts:135-139). In both, readyTimeoutMs is the post-handshake waitForRuntimeHostReady on a Host that is already connected. It is not a launch-election window. The local candidate path already passes electionDeadlineMs to connectOrSpawnRuntimeHost (:1046), which resolves caller value → MAKA_RUNTIME_HOST_ELECTION_DEADLINE_MS → 75s by itself (connect-or-spawn.ts:345-348). So the change does not touch the replace-and-reshuffle loop described in the commit message. What it does change is narrower and unstated: (a) remote/WSL Hosts that connect but never become ready now take 75s instead of 45s to fail, and (b) an env var named ELECTION_DEADLINE now also sets the remote/WSL readiness wait. For WSL, the in-distro election (if any) runs before the handshake, and that is still capped by the 45s DEFAULT_RUNTIME_HOST_WSL_STARTUP_TIMEOUT_MS handshake default unless the caller supplies handshakeTimeoutMs. Suggest either dropping this commit from the PR, or restating its rationale as "align remote/WSL readiness with the election window" with an explicit argument for that coupling. The helper and test are fine as code; the issue is what they are justified by.

...(input.peerClient === undefined ? {} : { peerClient: input.peerClient }),
...(input.refreshPeerRoutes === undefined
? {}
Expand Down
122 changes: 122 additions & 0 deletions packages/runtime-host/src/__tests__/managed-deployment.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ import {
} from '../operator/managed-deployment.js';
import { resolveRuntimeHostNpmDeploymentLayout } from '../operator/update-package-evidence.js';
import { connectOrSpawnRuntimeHostWithDependencies } from '../client/connect-or-spawn.js';
import type { OwnedCandidateAttempt } from '../client/launcher.js';
import { runtimeHostStartupError } from '../client/startup-error.js';
import { RuntimeHostPermanentReconnectError } from '../client/reconnect-lifecycle.js';
import {
Expand Down Expand Up @@ -470,6 +471,127 @@ test('managed reconnection classifies an unresponsive endpoint without authorizi
assert.equal(launches, 0);
});

const LOST_ATTEMPT_ID = '00000000-0000-4000-8000-000000000003';

function losingCandidateAttempt(): OwnedCandidateAttempt {
return {
pid: 4242,
startupAttemptId: LOST_ATTEMPT_ID,
exited: Promise.resolve({
code: 2,
signal: null,
stderr: '',
stderrTruncated: false,
}),
startupFailure: Promise.resolve({
reason: 'launch_election_lost',
startupAttemptId: LOST_ATTEMPT_ID,
}),
releaseToEnvironment() {},
async settle() {
return false;
},
};
}

// Issue #5843: a large transcript store makes the winning Candidate take longer
// than the election window to register. Every other candidate that is launched
// during that window loses the launch election and exits 2, and the Host never
// comes up. Losing the election is evidence that an owner is starting, so the
// election must spend its window waiting for that owner instead of reporting
// the loss as the Host's failure.
test('a candidate that loses the launch election is waited on, not replaced', async (t) => {
const input = await fixture(t);
let launches = 0;
let connectCalls = 0;
const startedAt = performance.now();
const result = await connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: input.capability.canonicalPath,
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate.js',
electionDeadlineMs: 300,
},
{
managedDeploymentAuthority: input.authority,
connectHost: async () => {
connectCalls += 1;
return { kind: 'unavailable', reason: 'not_registered', endpointConnected: false };
},
launchCandidate: () => {
launches += 1;
return { spawned: Promise.resolve(losingCandidateAttempt()) };
},
random: () => 0.5,
},
);
const elapsedMs = performance.now() - startedAt;

// Which of the two exits a lost election reaches depends on when the loser's
// diagnostic resolves relative to the window, so assert only the property
// that must hold on both: the loss never decides the outcome.
assert.equal(result.kind, 'failed');
if (result.kind !== 'failed') throw new Error('Expected the election to end in a failure');
assert.notEqual(
result.reason,
'launch_election_lost',
`election ended on another candidate's loss after ${Math.round(elapsedMs)}ms`,
);
assert.ok(launches >= 1);
});

test('a candidate that never wins is not permanent proof of a starting owner', async (t) => {
const input = await fixture(t);
let launches = 0;
const result = await connectOrSpawnRuntimeHostWithDependencies(
{
rootPath: input.capability.canonicalPath,
protocol: { min: RUNTIME_HOST_PROTOCOL_VERSION, max: RUNTIME_HOST_PROTOCOL_VERSION },
compositionId: INTERACTIVE_RUNTIME_HOST_COMPOSITION_ID,
candidateEntrypoint: 'candidate.js',
// Re-launching is floored at MIN_CANDIDATE_INTERVAL_MS, so the window has
// to be long enough for a second launch to be possible at all.
electionDeadlineMs: 900,
},
{
managedDeploymentAuthority: input.authority,
connectHost: async () => ({
kind: 'unavailable',
reason: 'not_registered',
endpointConnected: false,
}),
launchCandidate: () => {
launches += 1;
// A candidate that starts and never reports an election loss is the
// normal path: this client must keep launching it, not wait forever on
// an owner that does not exist.
return {
spawned: Promise.resolve({
pid: 9999,
exited: Promise.resolve({
code: 1,
signal: null,
stderr: '',
stderrTruncated: false,
}),
releaseToEnvironment() {},
async settle() {
return false;
},
}),
};
},
random: () => 0.5,
},
);

assert.equal(result.kind, 'failed');
if (result.kind !== 'failed') throw new Error('Expected the election to end in a failure');
assert.equal(result.reason, 'startup_timeout');
assert.ok(launches > 1, 'an election with no owner must still launch a Host');
});

test('concurrent install and unmanaged launch cannot both cross the authority boundary', async (t) => {
const input = await fixture(t);
const [claimResult, launchResult] = await Promise.allSettled([
Expand Down
46 changes: 37 additions & 9 deletions packages/runtime-host/src/client/connect-or-spawn.ts
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ import { abortable, waitForRuntimeHostReady } from './wait-for-ready.js';
// Candidate readiness includes the Windows named-pipe ACL helper, whose
// fail-closed ceiling is 60s. Leave enough room for election and connection
// bookkeeping after that helper returns.
const DEFAULT_ELECTION_DEADLINE_MS = 75_000;
export const DEFAULT_ELECTION_DEADLINE_MS = 75_000;
const DEFAULT_BACKOFF_MIN_MS = 20;
const DEFAULT_BACKOFF_MAX_MS = 250;
const MIN_CANDIDATE_INTERVAL_MS = 250;
Expand Down Expand Up @@ -459,13 +459,26 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
return result;
}
if (isPermanentCandidateStartupFailure(startupFailure) && pendingCandidateReports === 0) {
const selectedFailure = startupFailure;
electionSettled = true;
await selectCandidateStartupDiagnostic(
capability.rootId,
selectedFailure.startupAttemptId,
).catch(() => undefined);
return { kind: 'failed', reason: selectedFailure.reason };
if (isLaunchElectionLoss(startupFailure)) {
// Losing the launch election is not a Host failure. It is proof that
// another candidate holds this root and is still coming up, so the
// only honest reading is "keep waiting for the one that won".
// Returning here is what made a slow first startup look permanently
// unavailable: the desktop saw a failure a second after it asked,
// while the candidate that owned the root was still loading its
// transcript store, and every retry looked the same (issue #5843).
// The loser's startup diagnostic stays on disk, because that trace is
// what made the storm visible at all; it is simply no longer terminal.
startupFailure = undefined;
} else {
const selectedFailure = startupFailure;
electionSettled = true;
await selectCandidateStartupDiagnostic(
capability.rootId,
selectedFailure.startupAttemptId,
).catch(() => undefined);
return { kind: 'failed', reason: selectedFailure.reason };
}
}

const now = performance.now();
Expand Down Expand Up @@ -595,7 +608,7 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
await sleep(Math.min(remaining, Math.max(1, Math.round(backoffMs * jitter))), input.signal);
backoffMs = Math.min(DEFAULT_BACKOFF_MAX_MS, backoffMs * 2);
}
if (startupFailure) {
if (startupFailure && !isLaunchElectionLoss(startupFailure)) {
const selectedFailure = startupFailure;
electionSettled = true;
await selectCandidateStartupDiagnostic(
Expand Down Expand Up @@ -623,6 +636,21 @@ export async function connectOrSpawnRuntimeHostWithDependencies(
}
}

/**
* A Candidate that lost the launch election is evidence about *another*
* Candidate - it holds the root and is still coming up - so the loss must never
* decide how this election ends, whether the loop breaks on it or the window
* simply runs out. Failing on it made a slow first startup look like a
* permanently unavailable Host (issue #5843).
*/
function isLaunchElectionLoss(
failure: CandidateStartupFailureReport | undefined,
): failure is CandidateStartupFailureReport & {
readonly reason: 'launch_election_lost';
} {
return failure?.reason === 'launch_election_lost';
}

function recordElectionResult(
result: ElectionConnectionResult,
observations: MutableElectionObservations,
Expand Down
3 changes: 3 additions & 0 deletions packages/runtime-host/src/client/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,8 +157,11 @@ export {
type RuntimeHostConnectionCatalogSnapshot,
} from './catalog-reader.js';
export {
DEFAULT_ELECTION_DEADLINE_MS,
ELECTION_DEADLINE_MS_ENV_VAR,
IDLE_GRACE_MS_ENV_VAR,
connectOrSpawnRuntimeHost,
electionDeadlineMsFromEnvironment,
type CandidateExitDetails,
type ConnectOrSpawnRuntimeHostInput,
type ConnectOrSpawnRuntimeHostResult,
Expand Down