Skip to content

Commit 030a795

Browse files
committed
fix(core): a gracefulShutdown: false kernel no longer exits the process when its teardown times out
The timeout branch of shutdown() now reads the same predicate the constructor uses to install the signal listeners: a kernel that owns the process (true, the default) keeps the hard exit(1); a host-owned kernel (false) logs the timeout at error, stays stopped and returns to the host. The #5274 timeout pin moves to a true fixture; new pins hold the false half, the two-kernel shape and the non-timeout controls. Claude-Session: https://claude.ai/code/session_01EUBvqtauTDmHi2ZgY759p2 Co-authored-by: Claude <noreply@anthropic.com>
1 parent 16096e8 commit 030a795

3 files changed

Lines changed: 223 additions & 12 deletions

File tree

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
2+
3+
/**
4+
* A kernel whose teardown times out exits the process only if it OWNS the
5+
* process (#22335, ruling B): `gracefulShutdown` true — the default, the kernel
6+
* installed the signal listeners itself — keeps the hard `exit(1)` (pinned in
7+
* kernel.test.ts, the #5274 timeout pin). `gracefulShutdown` false means the
8+
* host owns the process: the kernel logs the timeout, marks itself `stopped`,
9+
* and `shutdown()` returns to the host, which decides whether the process ends.
10+
*
11+
* Before this, the timeout branch called `process.exit(1)` without reading the
12+
* option, so a host running several `false` kernels in one process (one per
13+
* environment) lost every one of them when one teardown hung.
14+
*
15+
* `process.exit` is stubbed in every test here: the behaviour under test is
16+
* whether the kernel ends the process, and a real `exit(1)` would end the
17+
* vitest worker with it.
18+
*/
19+
20+
import { describe, it, expect, beforeEach, afterEach, vi } from 'vitest';
21+
import { ObjectKernel } from './kernel';
22+
import type { Plugin } from './types';
23+
24+
type ExitSpy = { mock: { calls: unknown[][] }; mockRestore: () => void };
25+
type LogMethod = (message: string, ...rest: unknown[]) => void;
26+
27+
const SHUTDOWN_TIMEOUT_MS = 20;
28+
29+
describe('ObjectKernel shutdown timeout under gracefulShutdown: false', () => {
30+
let exitSpy: ExitSpy;
31+
32+
beforeEach(() => {
33+
exitSpy = vi.spyOn(process, 'exit').mockImplementation((() => undefined) as never) as unknown as ExitSpy;
34+
});
35+
36+
afterEach(() => {
37+
exitSpy.mockRestore();
38+
});
39+
40+
const hostOwnedKernel = () =>
41+
new ObjectKernel({
42+
logger: { level: 'silent' },
43+
gracefulShutdown: false,
44+
skipSystemValidation: true,
45+
shutdownTimeout: SHUTDOWN_TIMEOUT_MS,
46+
});
47+
48+
const spyOnError = (kernel: ObjectKernel) =>
49+
vi.spyOn((kernel as unknown as { logger: { error: LogMethod } }).logger, 'error');
50+
51+
/** A plugin whose `kernel:shutdown` subscriber holds teardown open until released. */
52+
const hungTeardown = (reached: string[]) => {
53+
let release!: () => void;
54+
const gate = new Promise<void>((resolve) => { release = resolve; });
55+
const plugin: Plugin = {
56+
name: 'hung-teardown',
57+
version: '1.0.0',
58+
init: async (ctx) => {
59+
ctx.hook('kernel:shutdown', async () => {
60+
await gate;
61+
reached.push('teardown-resumed');
62+
});
63+
},
64+
destroy: async () => { reached.push('destroy'); },
65+
};
66+
return { plugin, release: () => release() };
67+
};
68+
69+
it('logs the timeout at error, ends stopped, resolves and does NOT exit the process', async () => {
70+
const reached: string[] = [];
71+
const { plugin, release } = hungTeardown(reached);
72+
const kernel = hostOwnedKernel();
73+
const errorSpy = spyOnError(kernel);
74+
await kernel.use(plugin);
75+
await kernel.bootstrap();
76+
77+
try {
78+
await expect(kernel.shutdown()).resolves.toBeUndefined();
79+
80+
expect(exitSpy.mock.calls).toEqual([]);
81+
expect(kernel.getState()).toBe('stopped');
82+
const timeoutLines = errorSpy.mock.calls.filter((c) => String(c[0]).startsWith('Shutdown timed out'));
83+
expect(timeoutLines).toHaveLength(1);
84+
expect(timeoutLines[0][1]).toBeInstanceOf(Error);
85+
// The `true` line says an exit is being forced; under `false` none is.
86+
expect(errorSpy.mock.calls.map((c) => String(c[0]))).not.toContain('Shutdown timed out — forcing exit');
87+
// The teardown was abandoned, not finished.
88+
expect(reached).toEqual([]);
89+
} finally {
90+
release();
91+
}
92+
93+
// The hung teardown keeps running in the background. When it finishes,
94+
// the kernel does not act on it: still `stopped`, still no exit, and a
95+
// second `shutdown()` is the already-stopped no-op.
96+
await vi.waitFor(() => expect(reached).toEqual(['teardown-resumed', 'destroy']));
97+
expect(kernel.getState()).toBe('stopped');
98+
await expect(kernel.shutdown()).resolves.toBeUndefined();
99+
expect(exitSpy.mock.calls).toEqual([]);
100+
errorSpy.mockRestore();
101+
});
102+
103+
it('two host-owned kernels in one process: stopping the hung one leaves the other running and the process alive', async () => {
104+
const reached: string[] = [];
105+
const { plugin, release } = hungTeardown(reached);
106+
const hung = hostOwnedKernel();
107+
const sibling = hostOwnedKernel();
108+
await hung.use(plugin);
109+
await hung.bootstrap();
110+
await sibling.bootstrap();
111+
112+
try {
113+
await hung.shutdown();
114+
115+
expect(hung.getState()).toBe('stopped');
116+
expect(sibling.getState()).toBe('running');
117+
expect(exitSpy.mock.calls).toEqual([]);
118+
119+
// The sibling is untouched: the host can still stop it normally.
120+
await sibling.shutdown();
121+
expect(sibling.getState()).toBe('stopped');
122+
expect(exitSpy.mock.calls).toEqual([]);
123+
} finally {
124+
release();
125+
}
126+
});
127+
128+
// Control: a teardown that FAILS rather than hangs never exited the process
129+
// under `false`, before this change or after it. Both shapes: a throwing
130+
// `destroy()` (isolated inside the teardown) and an error escaping the
131+
// teardown altogether (the non-timeout branch of `shutdown()`'s catch).
132+
it.each([
133+
['a throwing destroy()', (kernel: ObjectKernel) =>
134+
kernel.use({
135+
name: 'throwing-destroy',
136+
version: '1.0.0',
137+
init: async () => {},
138+
destroy: async () => { throw new Error('destroy boom'); },
139+
})],
140+
['an error escaping the teardown', async (kernel: ObjectKernel) => {
141+
vi.spyOn(kernel as unknown as { performShutdown: () => Promise<void> }, 'performShutdown')
142+
.mockRejectedValue(new Error('teardown boom'));
143+
}],
144+
] as const)('control: %s does not exit the process and is not reported as a timeout', async (_label, arrange) => {
145+
const kernel = hostOwnedKernel();
146+
const errorSpy = spyOnError(kernel);
147+
await arrange(kernel);
148+
await kernel.bootstrap();
149+
150+
await expect(kernel.shutdown()).resolves.toBeUndefined();
151+
152+
expect(exitSpy.mock.calls).toEqual([]);
153+
expect(kernel.getState()).toBe('stopped');
154+
expect(errorSpy.mock.calls.some((c) => String(c[0]).startsWith('Shutdown timed out'))).toBe(false);
155+
errorSpy.mockRestore();
156+
});
157+
});

‎packages/core/src/kernel.test.ts‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -966,10 +966,17 @@ describe('ObjectKernel', () => {
966966
// making progress, so the process would hang holding whatever it failed
967967
// to release. This behaviour is unchanged by #5274 — pinned so the
968968
// narrowing of the catch cannot quietly take it along.
969+
//
970+
// The fixture owns the process (`gracefulShutdown: true`, the default):
971+
// only a kernel that installed the signal listeners itself exits on a
972+
// timeout (#22335). The `false` half — no exit, the host decides — is
973+
// pinned in kernel-shutdown-timeout-ownership.test.ts.
969974
it('still logs the timeout and still forces exit(1) when shutdown genuinely times out (#5274)', async () => {
975+
const signals = ['SIGINT', 'SIGTERM', 'SIGQUIT'] as const;
976+
const listenersBefore = new Map(signals.map((s) => [s, process.listeners(s)]));
970977
const slowKernel = new ObjectKernel({
971978
logger: { level: 'error' },
972-
gracefulShutdown: false,
979+
gracefulShutdown: true,
973980
skipSystemValidation: true,
974981
shutdownTimeout: 20,
975982
});
@@ -1000,6 +1007,13 @@ describe('ObjectKernel', () => {
10001007
} finally {
10011008
exitSpy.mockRestore();
10021009
errorSpy.mockRestore();
1010+
// A `true` kernel listens on the worker's own process; never
1011+
// leave its listeners behind, whatever the test did.
1012+
for (const s of signals) {
1013+
for (const l of process.listeners(s)) {
1014+
if (!listenersBefore.get(s)!.includes(l)) process.removeListener(s, l);
1015+
}
1016+
}
10031017
}
10041018
}, 5000);
10051019

‎packages/core/src/kernel.ts‎

Lines changed: 51 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -70,10 +70,29 @@ export interface ObjectKernelConfig {
7070
defaultStartupTimeout?: number;
7171

7272
/**
73-
* Whether to enable graceful shutdown. When true (the default) the kernel
74-
* listens for SIGINT, SIGTERM and SIGQUIT from construction until it stops:
75-
* a signal drains it and then exits the process. Its listeners are removed
76-
* when it reaches `stopped`, so a stopped kernel never handles a signal.
73+
* Whether this kernel owns the process. It decides two things, always
74+
* together:
75+
*
76+
* - **Who installs the signal listeners.** When true (the default) the
77+
* kernel listens for SIGINT, SIGTERM and SIGQUIT from construction until
78+
* it stops: a signal drains it and then exits the process. Its listeners
79+
* are removed when it reaches `stopped`, so a stopped kernel never
80+
* handles a signal. When false the kernel installs none, and the host
81+
* handles signals itself.
82+
* - **Who exits the process when a teardown times out.** When `shutdown()`
83+
* overruns `shutdownTimeout`, a kernel with this option true logs the
84+
* timeout and calls `process.exit(1)`. A kernel with it false does NOT
85+
* exit the process: it logs the timeout at `error`, marks itself
86+
* `stopped`, and `shutdown()` returns to the host, which decides whether
87+
* the process exits. The hung teardown is not cancelled — it keeps
88+
* running in the background and holds whatever it has not released yet,
89+
* until it finishes or the host ends the process.
90+
*
91+
* Set it false when something else owns the process: a server framework
92+
* that installs its own signal handlers, or a host that runs several
93+
* kernels in one process and must not lose all of them when one teardown
94+
* hangs. Such a host exits the process itself when it decides to.
95+
* `shutdown()` never rejects under either setting.
7796
*/
7897
gracefulShutdown?: boolean;
7998

@@ -599,13 +618,34 @@ export class ObjectKernel {
599618

600619
if (error === shutdownTimeoutError) {
601620
// GENUINE timeout: `performShutdown()` is still running and has
602-
// stopped making progress, so the process would otherwise hang
603-
// holding whatever it failed to release. Hard-exit stays — it
604-
// is the only branch it was ever right for.
605-
this.logger.error('Shutdown timed out — forcing exit', error as Error);
606-
// Flush logger then hard-exit; the process would otherwise hang
607-
await this.logger.destroy();
608-
process.exit(1);
621+
// stopped making progress. Whether that ends the process
622+
// depends on who owns it, and the test is the one the
623+
// constructor uses to install the signal listeners, so a kernel
624+
// exits on a timeout exactly when it took the signals itself
625+
// (`ObjectKernelConfig.gracefulShutdown`).
626+
if (this.config.gracefulShutdown) {
627+
// The kernel owns the process, which would otherwise hang
628+
// holding whatever the teardown failed to release.
629+
// Hard-exit stays — it is the only branch it was ever
630+
// right for.
631+
this.logger.error('Shutdown timed out — forcing exit', error as Error);
632+
// Flush logger then hard-exit; the process would otherwise hang
633+
await this.logger.destroy();
634+
process.exit(1);
635+
} else {
636+
// The host owns the process. Exiting here would end every
637+
// other kernel and workload it runs over one hung
638+
// teardown, so the kernel reports, stays `stopped` and
639+
// returns. Nothing here acts on the abandoned teardown
640+
// again: `performShutdown()` changes no state when it
641+
// finishes, and the kernel's own later log lines from it
642+
// reach the console only, the file stream having been
643+
// closed in `finally`.
644+
this.logger.error(
645+
`Shutdown timed out after ${this.config.shutdownTimeout}ms — the kernel is stopped but its teardown is still running in the background and may still hold what it has not released; gracefulShutdown is false, so the host owns the process and it is NOT being exited — the host decides whether it exits`,
646+
error as Error,
647+
);
648+
}
609649
} else {
610650
// NOT a timeout. `performShutdown()` isolates every teardown
611651
// step it owns (hook dispatch, each destroy(), each shutdown

0 commit comments

Comments
 (0)