feat(threads): run an isolated application in a dedicated worker thread

DESIGN.md

@@ -1955,10 +1955,10 @@ the table first), and who owns the column-family wrappers the declaration opens.
 An application that declared `branchedDatabases` declares through `scopedTableFactory(branches)`, which
 routes each declaration by database name — to the branch of that name, or to `table()` itself. GraphQL
 `@table` (`graphql.ts`), `scope.ensureTable` (`components/Scope.ts`, `componentLoader.ts`) and
-`defineTable` (`defineTableUsing`, through `security/jsLoader.ts`) all go through it. **An unbranched
-application gets `table` and `defineTable` by identity** — `scopedTableFactory(undefined) === table` —
-so the request path of every application that does not branch is untouched; only a branched
-application pays for the routing, and only at declaration time.
+`defineTable` (`defineTableUsing`, through `security/jsLoader.ts`) all go through it. **An unbranched,
+shared application gets `table` and `defineTable` by identity** — `scopedTableFactory(undefined) ===
+table`. An isolated application gets a declaration-only wrapper even without branches, so its own
+schema can claim maintenance that no pool worker configured; hydrated tables remain pool-owned.
 
 Consequences to preserve:
 

bin/restart.ts

@@ -10,6 +10,8 @@ import {
 	beginProcessShutdown,
 	restartWorkers,
 	isThreadRunning,
+	decodeRestartScope,
+	getRunningIsolatedApplications,
 	onMessageByType,
 	shutdownWorkersNow,
 } from '../server/threads/manageThreads.js';
@@ -26,6 +28,7 @@ envMgr.initSync();
 
 const RESTART_RESPONSE = `Restarting Harper. This may take up to ${hdbTerms.RESTART_TIMEOUT_MS / 1000} seconds.`;
 const INVALID_SERVICE_ERR = 'Invalid service';
+const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000;
 
 let calledFromCli;
 
@@ -35,8 +38,15 @@ export { restart, restartService };
 if (isMainThread) {
 	onMessageByType(hdbTerms.ITC_EVENT_TYPES.RESTART, async (message, port) => {
 		try {
-			if (message.removeBranchesFor) await restartThenRemoveBranches(message.workerType, message.removeBranchesFor);
-			else if (message.workerType) await restartService({ service: message.workerType });
+			// `scope` stays in its wire form ('' pool, a name, absent = all) until restartService decodes it once
+			if (message.removeBranchesFor)
+				await restartThenRemoveBranches(message.workerType, message.removeBranchesFor, message.scope);
+			else if (message.workerType)
+				await restartService({
+					service: message.workerType,
+					scope: message.scope,
+					scopeFallback: message.scopeFallback,
+				});
 			else restart({ operation: 'restart' });
 		} finally {
 			port.postMessage({ type: 'restart-complete' });
@@ -48,13 +58,13 @@ if (isMainThread) {
  * Restart, then remove the branches of an application dropped on a worker (which cannot outlive the
  * restart it asked for). The restart happens even if the component lock cannot be taken.
  */
-async function restartThenRemoveBranches(service: string, project: string): Promise<void> {
+async function restartThenRemoveBranches(service: string, project: string, scope: string | undefined): Promise<void> {
 	let restarted = false;
 	const restartHttpWorkers = async () => {
 		restarted = true;
 		processMan.expectedRestartOfChildren();
 		hdbLogger.notify('Restarting http_workers');
-		return restartWorkers('http');
+		return restartWorkers('http', undefined, true, null, decodeRestartScope({ scope }));
 	};
 	try {
 		const componentPath = path.join(getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT) as string, project);
@@ -89,7 +99,7 @@ async function restartThenRemoveBranches(service: string, project: string): Prom
 		);
 	} catch (error) {
 		hdbLogger.error(`Could not remove the branched database storage of ${project}`, error);
-		if (!restarted) await restartService({ service });
+		if (!restarted) await restartService({ service, scope });
 	}
 }
 
@@ -183,6 +193,49 @@ async function restartService(req: any) {
 	if (hdbTerms.HDB_PROCESS_SERVICES[service] === undefined) {
 		throw handleHDBError(new Error(), INVALID_SERVICE_ERR, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
 	}
+	const requestedScope = decodeRestartScope(req);
+	if (requestedScope !== undefined && typeof requestedScope !== 'string') {
+		throw handleHDBError(
+			new Error(),
+			'Invalid HTTP worker restart scope: expected a string',
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+	let fallbackScope;
+	if (req.scopeFallback !== undefined) {
+		fallbackScope = decodeRestartScope({ scope: req.scopeFallback });
+		if (fallbackScope !== undefined) {
+			throw handleHDBError(
+				new Error(),
+				'Invalid HTTP worker restart scope fallback: expected the pool scope',
+				HTTP_STATUS_CODES.BAD_REQUEST,
+				undefined,
+				undefined,
+				true
+			);
+		}
+	}
+	if (typeof requestedScope === 'string' && requestedScope !== '*' && req.scopeFallback === undefined) {
+		envMgr.initSync(true);
+		const { isIsolatedApplication } = await import('../server/threads/isolatedApplications.ts');
+		const configured = isIsolatedApplication(requestedScope);
+		const runningApplications = configured
+			? []
+			: await getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+		if (!configured && !runningApplications.includes(requestedScope)) {
+			throw handleHDBError(
+				new Error(),
+				`Unknown isolated application restart scope: ${requestedScope}`,
+				HTTP_STATUS_CODES.BAD_REQUEST,
+				undefined,
+				undefined,
+				true
+			);
+		}
+	}
 	processMan.expectedRestartOfChildren();
 	if (!isMainThread) {
 		if (req.replicated) {
@@ -191,6 +244,8 @@ async function restartService(req: any) {
 		parentPort.postMessage({
 			type: hdbTerms.ITC_EVENT_TYPES.RESTART,
 			workerType: service,
+			scope: req.scope, // wire form, forwarded as received
+			scopeFallback: req.scopeFallback,
 		});
 		parentPort.ref(); // don't let the parent thread exit until we're done
 		await new Promise<void>((resolve) => {
@@ -265,7 +320,12 @@ async function restartService(req: any) {
 			if (calledFromCli) {
 				await processMan.restart(hdbTerms.PROCESS_DESCRIPTORS.HDB);
 			} else {
-				await restartWorkers('http');
+				let scope = requestedScope;
+				if (req.scopeFallback !== undefined && typeof scope === 'string' && scope !== '*') {
+					const runningApplications = await getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+					if (!runningApplications.includes(scope)) scope = fallbackScope;
+				}
+				await restartWorkers('http', undefined, true, null, scope);
 			}
 			break;
 		default:

components/Application.ts

@@ -84,6 +84,8 @@ interface ApplicationConfig {
 	 * Per-application globals are a property of thread-level isolation, not of branching.
 	 */
 	branchedDatabases?: string[] | true;
+	/** Run in a worker thread of its own that loads no other application. */
+	isolated?: boolean;
 	// an application config can have other arbitrary properties
 	[key: string]: unknown;
 }
@@ -218,6 +220,15 @@ export function assertApplicationConfig(
 		}
 	}
 	assertBranchedDatabases(applicationName, applicationConfig.branchedDatabases);
+	assertIsolationConfig(applicationName, applicationConfig.isolated);
+}
+
+export function assertIsolationConfig(applicationName: string, isolated: unknown): void {
+	if (isolated !== undefined && typeof isolated !== 'boolean') {
+		throw new TypeError(
+			`Invalid 'isolated' for application ${applicationName}: expected a boolean, got ${typeof isolated}`
+		);
+	}
 }
 
 /**
@@ -3444,6 +3455,7 @@ export function shouldPackLocalDirectory(packageIdentifier: string | undefined,
  * @returns A promise that resolves when all preparation steps complete.
  */
 export type PrepareApplicationOptions = {
+	beforePrepare?: () => Promise<void>;
 	/**
 	 * Runs against the built candidate while the live version is still serving, and BEFORE the swap. A
 	 * throw here means the candidate never goes live — which is the whole difference from the previous
@@ -3460,6 +3472,7 @@ export async function prepareApplication(application: Application, options: Prep
 		await withComponentPreparationLock(
 			application.dirPath,
 			async () => {
+				await options.beforePrepare?.();
 				const asideStagingDir = extractionStagingDirectory(application.dirPath);
 				let recoveryPending = true;
 				try {

components/Scope.ts

@@ -20,6 +20,7 @@ import {
 } from './componentSecrets.ts';
 import type { SecretsView } from './componentSecrets.ts';
 import { deployLifecycle } from './deployLifecycle.ts';
+import { thisThreadOwnsApplication } from '../server/threads/isolatedApplications.ts';
 
 export class MissingDefaultFilesOptionError extends Error {
 	constructor() {
@@ -241,7 +242,10 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 
 	ensureTable<TableResourceType = unknown>(options: any): TableResourceType {
 		options.origin = this.#origin;
-		return scopedTableFactory(this.applicationScope?.branches)<TableResourceType>(options);
+		return scopedTableFactory(
+			this.applicationScope?.branches,
+			thisThreadOwnsApplication(this.applicationScope?.name)
+		)<TableResourceType>(options);
 	}
 
 	#handleOptionsWatcherReady(): void {

components/componentLoader.ts

@@ -32,8 +32,13 @@ import { restartWorkers, getWorkerIndex } from '../server/threads/manageThreads.
 import { resetRestartNeeded, subscribeToRestartRequests } from './requestRestart.ts';
 import { trackScopeClose } from './scopeShutdown.ts';
 import { deployLifecycle } from './deployLifecycle.ts';
-import { assertBranchedDatabases } from './Application.ts';
+import { assertBranchedDatabases, assertIsolationConfig } from './Application.ts';
 import { prepareBranches } from '../resources/branchDatabase.ts';
+import {
+	isIsolatedApplication,
+	shouldLoadApplicationHere,
+	thisThreadOwnsApplication,
+} from '../server/threads/isolatedApplications.ts';
 import { toScopeMount, nestScopeMount, type ScopeMount } from './scopeMount.ts';
 import { scopedImport } from '../security/jsLoader.ts';
 import { server } from '../server/Server.ts';
@@ -161,6 +166,32 @@ function rootConfigBranchedDatabases(appName: string): string[] | true | undefin
 	return (getConfigObj()?.[appName] as any)?.branchedDatabases;
 }
 
+/**
+ * Whether this thread is the one that loads `appName` (harper#642, tier 2). An isolated application
+ * is loaded by its dedicated worker and by nothing else; every other thread loads only the
+ * applications that are not isolated. Decided before any of the application's modules are imported.
+ * With no worker threads at all there is nowhere for an isolated application to run, so it fails
+ * closed here rather than silently sharing the only thread.
+ */
+function placedOnThisThread(appName: string): boolean {
+	if (Object.hasOwn(TRUSTED_RESOURCE_PLUGINS, appName)) return true;
+	try {
+		assertIsolationConfig(appName, (getConfigObj()?.[appName] as any)?.isolated);
+	} catch (error) {
+		componentLifecycle.failed(appName, error, `Component '${appName}' failed to load`);
+		return false;
+	}
+	if (isIsolatedApplication(appName) && isMainThread && getWorkerIndex() === 0) {
+		componentLifecycle.failed(
+			appName,
+			new Error(`Application '${appName}' is isolated, which needs a worker thread of its own, but threads.count is 0`),
+			`Component '${appName}' failed to load`
+		);
+		return false;
+	}
+	return shouldLoadApplicationHere(appName);
+}
+
 /**
  * Resolves the mount for `appName`, or reports the failure and returns `undefined` if the
  * configured `host`/`urlPath` is invalid. The caller must skip loading the application in that
@@ -268,6 +299,7 @@ export async function loadComponentDirectories(
 						}
 						return;
 					}
+					if (!placedOnThisThread(appName)) return;
 					const mountResult = tryRootConfigMount(appName);
 					if (!mountResult.ok) return;
 					const loadedModules = new Set<any>();
@@ -316,6 +348,7 @@ export async function loadComponentDirectories(
 				);
 				continue;
 			}
+			if (!placedOnThisThread(appName)) continue;
 			const appFolder = join(CF_ROUTES_DIR, appName);
 			const mountResult = tryRootConfigMount(appName);
 			if (!mountResult.ok) continue;
@@ -347,7 +380,7 @@ export async function loadComponentDirectories(
 	if (hdbAppFolder) {
 		if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder);
 		const mountResult = tryRootConfigMount(basename(hdbAppFolder));
-		if (mountResult.ok) {
+		if (mountResult.ok && placedOnThisThread(basename(hdbAppFolder))) {
 			cfsLoaded.push(
 				serializeComponentLoad(hdbAppFolder, () =>
 					loadComponent(hdbAppFolder, cycleResources, hdbAppFolder, {
@@ -849,6 +882,9 @@ export async function loadComponent(
 			if (!componentConfig) continue;
 
 			// Initialize loading status for all components (applications and extensions)
+			// A root-config application (`package:`) is placed like a directory one: an isolated application
+			// loads only in its dedicated worker, and that worker loads no other application.
+			if (isRoot && componentConfig.package && !placedOnThisThread(componentName)) continue;
 			componentLifecycle.loading(componentStatusName);
 
 			const subApplicationScope = isRoot
@@ -940,7 +976,10 @@ export async function loadComponent(
 				// our own trusted modules can be directly retrieved from our map, otherwise use the (configurable) secure module loader
 				const ensureTable = (options: any) => {
 					options.origin = origin;
-					return scopedTableFactory(applicationScope.branches)(options);
+					return scopedTableFactory(
+						applicationScope.branches,
+						thisThreadOwnsApplication(applicationScope.name)
+					)(options);
 				};
 				// call the main start hook
 				const network =
@@ -1160,7 +1199,7 @@ export async function loadComponent(
 						// — not abort the loop and discard a pending follow-up, like the save that fixes it.
 						try {
 							await loadComponentDirectories();
-							await restartWorkers();
+							await restartWorkers(undefined, undefined, true, null, '*');
 						} catch (error) {
 							harperLogger.error('Error during component reload', error);
 						}