Fall back to polling-based file watching on inotify/FD exhaustion

components/EntryHandler.ts

@@ -9,6 +9,11 @@ import { readFile } from 'node:fs/promises';
 import { FilesOption } from './deriveGlobOptions.ts';
 import { deriveURLPath } from './deriveURLPath.ts';
 import { isMatch } from 'micromatch';
+import {
+	DIRECTORY_POLLING_FALLBACK_OPTIONS,
+	isWatcherExhaustionError,
+	warnWatcherFallback,
+} from '../utility/watcherFallback.ts';
 
 export interface BaseEntry {
 	stats?: Stats;
@@ -85,6 +90,9 @@ export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
 	// chokidar instance — otherwise a rapid pause→resume can overlap teardown
 	// and setup, which under inotify pressure can produce spurious EMFILE.
 	#pausedClose?: Promise<void>;
+	#usingPolling: boolean = false;
+	#closed: boolean = false;
+	#openCount: number = 0;
 	ready: Promise<any[]>;
 
 	constructor(name: string, directory: string, config: FilesOption | FileAndURLPathConfig, logger?: Logger) {
@@ -169,6 +177,25 @@ export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
 	}
 
 	#handleError(error: unknown): void {
+		if (isWatcherExhaustionError(error)) {
+			// Swallow every exhaustion error — chokidar can emit several before the
+			// failed native watcher closes, and we don't want a flurry of ENOSPC to
+			// surface to consumers in the middle of recovery.
+			if (!this.#usingPolling) {
+				warnWatcherFallback(this.#component.directory);
+				this.#usingPolling = true;
+				// Reopen with polling. #watch() itself guards against reopen-after-close.
+				// The .catch is required because #watch() internally awaits the failed
+				// watcher's close(), which can reject under the same FD/inotify pressure
+				// that triggered this path; without it Node would treat that as an
+				// unhandled rejection (matches the .catch pattern used in
+				// OptionsWatcher / RootConfigWatcher).
+				this.#watch().catch(() => {
+					// Teardown errors on an already-failed watcher are not actionable.
+				});
+			}
+			return;
+		}
 		this.emit('error', error);
 	}
 
@@ -201,6 +228,10 @@ export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
 		await this.#watcher?.close();
 		this.#watcher = undefined;
 
+		// If close() landed while a previous close()/recreate was awaiting, don't
+		// install a fresh watcher — it would outlive the EntryHandler.
+		if (this.#closed) return this.ready;
+
 		// pause() may have landed in the gap before our async close resolved.
 		// If so, do not install a replacement watcher — resume() will.
 		if (this.#paused) return this.ready;
@@ -212,11 +243,13 @@ export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
 
 		const allowedBases = this.#component.patternBases.map((base) => join(this.#component.directory, base));
 
+		this.#openCount++;
 		this.#watcher = chokidar
 			.watch(this.#component.commonPatternBase, {
 				cwd: this.#component.directory,
 				persistent: false,
 				followSymlinks: false,
+				...(this.#usingPolling ? DIRECTORY_POLLING_FALLBACK_OPTIONS : {}),
 				ignored: (path) => {
 					const normalizedPath = path.replace(/\\/g, '/');
 					const normalizedBases = allowedBases.map((base) => base.replace(/\\/g, '/'));
@@ -246,7 +279,27 @@ export class EntryHandler extends EventEmitter<EntryHandlerEventMap> {
 		return this.ready;
 	}
 
+	// Test-only: simulate the underlying chokidar watcher emitting an error.
+	// Exposed so the polling-fallback path can be exercised without triggering a
+	// real ENOSPC/EMFILE on the host.
+	_simulateWatcherErrorForTests(error: unknown): void {
+		this.#handleError(error);
+	}
+
+	// Test-only: whether the watcher has fallen back to polling.
+	get _usingPollingForTests(): boolean {
+		return this.#usingPolling;
+	}
+
+	// Test-only: number of times the underlying watcher has been (re)opened.
+	// Used to assert that a close()-during-fallback race didn't install a
+	// replacement watcher.
+	get _openCountForTests(): number {
+		return this.#openCount;
+	}
+
 	close(): this {
+		this.#closed = true;
 		this.#watcher?.close();
 		this.#watcher = undefined;
 

components/OptionsWatcher.ts

@@ -7,6 +7,7 @@ import { readFile } from 'node:fs/promises';
 import { isDeepStrictEqual } from 'util';
 import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts';
 import { cloneDeep } from 'lodash';
+import { POLLING_FALLBACK_OPTIONS, isWatcherExhaustionError, warnWatcherFallback } from '../utility/watcherFallback.ts';
 
 export interface Config {
 	[key: string]: ConfigValue;
@@ -82,21 +83,34 @@ export class CannotSetPropertyError extends Error {
  */
 export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 	#filePath: string;
-	#watcher: FSWatcher;
+	#watcher!: FSWatcher;
 	#scopedConfig?: ConfigValue;
 	#rootConfig?: Config;
 	#name: string;
 	#logger: Logger;
+	#usingPolling: boolean;
+	#closed: boolean;
+	#openCount: number = 0;
 	ready: Promise<any[]>;
 
 	constructor(name: string, filePath: string, logger?: Logger) {
 		super();
 		this.#name = name;
 		this.#filePath = filePath;
 		this.#logger = logger || loggerWithTag(name);
+		this.#usingPolling = false;
+		this.#closed = false;
 		this.ready = once(this, 'ready');
+		this.#openWatcher();
+	}
+
+	#openWatcher() {
+		this.#openCount++;
 		this.#watcher = chokidar
-			.watch(filePath, { persistent: false })
+			.watch(this.#filePath, {
+				persistent: false,
+				...(this.#usingPolling ? POLLING_FALLBACK_OPTIONS : {}),
+			})
 			.on('add', this.#handleChange.bind(this))
 			.on('change', this.#handleChange.bind(this))
 			.on('error', this.#handleError.bind(this))
@@ -150,6 +164,28 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 	}
 
 	#handleError(error: unknown) {
+		if (isWatcherExhaustionError(error)) {
+			// Swallow every exhaustion error — chokidar can emit several before the
+			// failed native watcher closes, and we don't want a flurry of ENOSPC to
+			// surface to consumers in the middle of recovery.
+			if (!this.#usingPolling) {
+				warnWatcherFallback(this.#filePath);
+				this.#usingPolling = true;
+				// Close the failed native watcher and reopen with polling. Guard
+				// against reopen-after-close: the caller may have invoked close()
+				// while this teardown was in flight. The .catch is required because
+				// `finally` would re-raise a teardown rejection as an unhandled one.
+				this.#watcher
+					.close()
+					.catch(() => {
+						// Teardown errors on an already-failed watcher are not actionable.
+					})
+					.finally(() => {
+						if (!this.#closed) this.#openWatcher();
+					});
+			}
+			return;
+		}
 		this.emit('error', new OptionsWatcherConfigFileError(this.#filePath, error));
 	}
 
@@ -283,10 +319,30 @@ export class OptionsWatcher extends EventEmitter<OptionsWatcherEventMap> {
 		this.emit('change', keys, value, this.#scopedConfig);
 	}
 
+	// Test-only: simulate the underlying chokidar watcher emitting an error.
+	// Exposed so the polling-fallback path can be exercised without triggering a
+	// real ENOSPC/EMFILE on the host.
+	_simulateWatcherErrorForTests(error: unknown): void {
+		this.#handleError(error);
+	}
+
+	// Test-only: whether the watcher has fallen back to polling.
+	get _usingPollingForTests(): boolean {
+		return this.#usingPolling;
+	}
+
+	// Test-only: number of times the underlying watcher has been (re)opened.
+	// Used to assert that a close()-during-fallback race didn't install a
+	// replacement watcher.
+	get _openCountForTests(): number {
+		return this.#openCount;
+	}
+
 	/**
 	 * Closes the underlying file watcher, emits the `close` event, and removes any listeners on the OptionsWatcher instance
 	 */
 	close() {
+		this.#closed = true;
 		this.#watcher.close();
 
 		this.emit('close');

config/RootConfigWatcher.ts

@@ -3,25 +3,78 @@ import { readFile } from 'node:fs/promises';
 import { getConfigFilePath } from './configUtils.js';
 import { EventEmitter, once } from 'node:events';
 import { parse } from 'yaml';
+import { POLLING_FALLBACK_OPTIONS, isWatcherExhaustionError, warnWatcherFallback } from '../utility/watcherFallback.ts';
 
 export class RootConfigWatcher extends EventEmitter {
 	#configFilePath: string;
-	#watcher: FSWatcher;
+	#watcher!: FSWatcher;
 	#config: any;
+	#usingPolling: boolean;
+	#closed: boolean;
+	#openCount: number = 0;
 	ready: Promise<any[]>;
 
 	constructor() {
 		super();
 		this.#configFilePath = getConfigFilePath();
+		this.#usingPolling = false;
+		this.#closed = false;
 		this.ready = once(this, 'ready');
+		this.#openWatcher();
+	}
+
+	#openWatcher() {
+		this.#openCount++;
 		this.#watcher = chokidar
-			.watch(this.#configFilePath, { persistent: false })
+			.watch(this.#configFilePath, {
+				persistent: false,
+				...(this.#usingPolling ? POLLING_FALLBACK_OPTIONS : {}),
+			})
 			.on('add', this.handleChange.bind(this))
 			.on('change', this.handleChange.bind(this))
 			.on('error', this.handleError.bind(this));
 	}
 
+	// Test-only: simulate the underlying chokidar watcher emitting an error.
+	// Exposed so the polling-fallback path can be exercised without triggering a
+	// real ENOSPC/EMFILE on the host.
+	_simulateWatcherErrorForTests(error: unknown): void {
+		this.handleError(error);
+	}
+
+	// Test-only: whether the watcher has fallen back to polling.
+	get _usingPollingForTests(): boolean {
+		return this.#usingPolling;
+	}
+
+	// Test-only: number of times the underlying watcher has been (re)opened.
+	get _openCountForTests(): number {
+		return this.#openCount;
+	}
+
 	handleError(error: unknown) {
+		if (isWatcherExhaustionError(error)) {
+			// Swallow every exhaustion error — chokidar can emit several before the
+			// failed native watcher closes, and we don't want a flurry of ENOSPC to
+			// surface to consumers in the middle of recovery.
+			if (!this.#usingPolling) {
+				warnWatcherFallback(this.#configFilePath);
+				this.#usingPolling = true;
+				// Guard against reopen-after-close: the caller may have invoked
+				// close() while this teardown was in flight. The .catch is required
+				// because `finally` would re-raise a teardown rejection as an
+				// unhandled one.
+				this.#watcher
+					.close()
+					.catch(() => {
+						// Teardown errors on an already-failed watcher are not actionable.
+					})
+					.finally(() => {
+						if (!this.#closed) this.#openWatcher();
+					});
+			}
+			return;
+		}
 		this.emit('error', error);
 	}
 
@@ -46,6 +99,7 @@ export class RootConfigWatcher extends EventEmitter {
 	}
 
 	close() {
+		this.#closed = true;
 		this.#watcher.close();
 		this.#config = undefined;
 		this.emit('close');

unitTests/components/EntryHandler.test.js

@@ -584,4 +584,104 @@ describe('EntryHandler', () => {
 			rmSync(directory, { recursive: true, force: true });
 		});
 	});
+
+	describe('polling fallback on watcher exhaustion', () => {
+		// harper#488: when the underlying chokidar watcher emits ENOSPC/EMFILE,
+		// the EntryHandler should re-open with polling rather than surfacing the
+		// error to consumers. EntryHandler's recovery path uses #watch() (which
+		// awaits the old watcher's close then checks #closed), structurally
+		// different from OptionsWatcher/RootConfigWatcher's explicit
+		// close().catch().finally() chain — so it needs its own coverage.
+
+		it('falls back to polling on ENOSPC and continues to receive change events', async () => {
+			const { directory } = createFixture(['a.txt']);
+			const entryHandler = new EntryHandler(basename(directory), directory, '**/*');
+			await entryHandler.ready;
+
+			const errorSpy = spy();
+			entryHandler.on('error', errorSpy);
+
+			assert.equal(entryHandler._usingPollingForTests, false);
+
+			entryHandler._simulateWatcherErrorForTests(Object.assign(new Error('boom'), { code: 'ENOSPC' }));
+
+			// Allow the close+reopen-with-polling to settle.
+			await waitFor(() => entryHandler._usingPollingForTests === true, 2000);
+			assert.equal(entryHandler._usingPollingForTests, true);
+			assert.equal(errorSpy.callCount, 0, 'ENOSPC should be swallowed');
+
+			// The polling watcher should pick up subsequent file writes; default
+			// directory polling interval is 3s, so allow up to 5s.
+			const addSpy = spy();
+			entryHandler.on('add', addSpy);
+			await writeFile(join(directory, 'b.txt'), 'b');
+			await waitFor(() => addSpy.callCount >= 1, 5000);
+			assert.ok(addSpy.callCount >= 1, 'polling watcher should fire add');
+
+			entryHandler.close();
+			rmSync(directory, { recursive: true, force: true });
+		}).timeout(8000);
+
+		it('propagates non-exhaustion errors and does not fall back', async () => {
+			const { directory } = createFixture(['a.txt']);
+			const entryHandler = new EntryHandler(basename(directory), directory, '**/*');
+			await entryHandler.ready;
+
+			const errorSpy = spy();
+			entryHandler.on('error', errorSpy);
+
+			entryHandler._simulateWatcherErrorForTests(Object.assign(new Error('boom'), { code: 'EACCES' }));
+			await new Promise((r) => setTimeout(r, 20));
+
+			assert.equal(entryHandler._usingPollingForTests, false);
+			assert.equal(errorSpy.callCount, 1, 'non-exhaustion error should propagate');
+
+			entryHandler.close();
+			rmSync(directory, { recursive: true, force: true });
+		});
+
+		it('swallows additional exhaustion errors during recovery', async () => {
+			const { directory } = createFixture(['a.txt']);
+			const entryHandler = new EntryHandler(basename(directory), directory, '**/*');
+			await entryHandler.ready;
+
+			const errorSpy = spy();
+			entryHandler.on('error', errorSpy);
+
+			const enospc = () => Object.assign(new Error('boom'), { code: 'ENOSPC' });
+			entryHandler._simulateWatcherErrorForTests(enospc());
+			entryHandler._simulateWatcherErrorForTests(enospc());
+			entryHandler._simulateWatcherErrorForTests(Object.assign(new Error('boom'), { code: 'EMFILE' }));
+
+			await waitFor(() => entryHandler._usingPollingForTests === true, 1000);
+			assert.equal(errorSpy.callCount, 0, 'all exhaustion errors should be swallowed');
+
+			entryHandler.close();
+			rmSync(directory, { recursive: true, force: true });
+		});
+
+		it('does not reopen watcher if close() is called during recovery', async () => {
+			// EntryHandler's recovery path: #handleError calls `void this.#watch()`,
+			// which does `await this.#watcher?.close(); if (this.#closed) return`.
+			// If close() lands between the ENOSPC and #watch()'s post-await check,
+			// the new chokidar watcher must not be installed.
+			const { directory } = createFixture(['a.txt']);
+			const entryHandler = new EntryHandler(basename(directory), directory, '**/*');
+			await entryHandler.ready;
+
+			assert.equal(entryHandler._openCountForTests, 1, 'one initial open');
+
+			// Trigger the fallback path, then immediately close before the inner
+			// `await this.#watcher.close()` resolves.
+			entryHandler._simulateWatcherErrorForTests(Object.assign(new Error('boom'), { code: 'ENOSPC' }));
+			entryHandler.close();
+
+			// Allow plenty of time for the would-be reopen to (not) happen.
+			await new Promise((r) => setTimeout(r, 200));
+
+			assert.equal(entryHandler._openCountForTests, 1, 'reopen must be suppressed by the close-during-fallback guard');
+
+			rmSync(directory, { recursive: true, force: true });
+		}).timeout(2000);
+	});
 });