Read users and roles through the record cache, ending user-cache refresh storms

DESIGN.md

@@ -100,6 +100,7 @@ Index of the design notes for the harper core: one line per note, grouped by the
 - [TLS hot-reload: cert vs. private key follow two different propagation paths (`security/keys.ts`)](security/DESIGN.md#tls-hot-reload-cert-vs-private-key-follow-two-different-propagation-paths-securitykeysts) — Certificates propagate through `hdb_certificate` subscriptions, private keys through each worker's own file watch; the two must reconverge.
 - [A component-facing export needs BOTH `index.ts` and `getHarperExports` (`security/jsLoader.ts`, `index.ts`)](security/DESIGN.md#a-component-facing-export-needs-both-indexts-and-getharperexports-securityjsloaderts-indexts) — A compartment resolves `harper` to `getHarperExports()`, not to `index.ts`; a value in only one list fails at component load, and only a fixture importing from `'harper'` catches it.
 - [Authentication converts every principal-resolution failure into a decision (`security/auth.ts`)](security/DESIGN.md#authentication-converts-every-principal-resolution-failure-into-a-decision-securityauthts) — A failed `server.getUser` becomes an in-place 401 or a deferred rejection, never a throw; an in-place answer must be recorded or a WebSocket/MQTT upgrade proceeds with no principal.
+- [User and role lookups read the records, and nothing derived from them outlives them (`security/user.ts`)](security/DESIGN.md#user-and-role-lookups-read-the-records-and-nothing-derived-from-them-outlives-them-securityuserts) — No per-thread copy and no change broadcast; derived data and cached principals are validated against entry versions.
 
 ## components/ — deploys and the load lifecycle
 

components/mcp/listChanged.ts

@@ -57,7 +57,9 @@ export function _resetListChangedForTest(): void {
 function loadItcHandlers(): ItcHandlers | undefined {
 	if (_itcHandlersOverride) return _itcHandlersOverride;
 	try {
-		return require('../../server/itc/serverHandlers');
+		const { schemaHandler, resourceHandler } = require('../../server/itc/serverHandlers');
+		const { onUserChange } = require('../../security/user');
+		return { schemaHandler, resourceHandler, userHandler: { addListener: onUserChange } };
 	} catch (err) {
 		harperLogger.trace(`MCP listChanged: ITC handlers unavailable (${(err as Error).message})`);
 		return undefined;
@@ -313,9 +315,9 @@ export function initListChanged(): boolean {
 	if (!handlers) return false;
 	let installed = 0;
 	if (handlers.userHandler?.addListener) {
-		// Harper's ITC handler treats listeners as `() => void`. Our handler is
-		// async (re-resolves users); fire-and-forget with a swallow so a rejection
-		// can never escape the event emitter as an UnhandledPromiseRejection.
+		// Listeners are `() => void`. Our handler is async (re-resolves users);
+		// fire-and-forget with a swallow so a rejection can never escape as an
+		// UnhandledPromiseRejection.
 		onUserChangeBound = () => {
 			onUserChange().catch((err) => harperLogger.trace(`MCP listChanged onUserChange: ${(err as Error).message}`));
 		};

integrationTests/security/fixtures/user-change-across-workers/config.yaml

@@ -0,0 +1,3 @@
+jsResource:
+  files: resources.js
+rest: true

integrationTests/security/fixtures/user-change-across-workers/resources.js

@@ -0,0 +1,12 @@
+import { threadId } from 'node:worker_threads';
+
+export class WhoAmI extends Resource {
+	static loadAsInstance = false;
+	allowRead() {
+		return true;
+	}
+	get() {
+		const user = this.getContext()?.user;
+		return { threadId, username: user?.username ?? null, superUser: user?.role?.permission?.super_user === true };
+	}
+}

integrationTests/security/user-change-across-workers.test.ts

@@ -0,0 +1,113 @@
+/**
+ * A user or role change holds on every HTTP worker once the operation that made it returns, though no
+ * worker is told about it: authentication reads hdb_user/hdb_role through the record cache and checks
+ * each cached credential against those records' versions (security/user.ts). Every worker's
+ * authorization cache is primed first, so a worker serving a stale cached principal fails the suite.
+ */
+import { suite, test, before, after } from 'node:test';
+import { strictEqual, ok } from 'node:assert';
+import { resolve } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing';
+// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine
+import { createApiClient } from '../apiTests/utils/client.mjs';
+import { WORKER_COUNT, NO_FULL_WORKER_COVERAGE, assertEveryWorkerStarted } from '../database/recordCachingWorkers.ts';
+import { fetchOnNewConnection, observeEveryWorker } from '../utils/connectionPerRequest.ts';
+
+const FIXTURE_PATH = resolve(import.meta.dirname, 'fixtures', 'user-change-across-workers');
+const ROLE = 'ucaw_role';
+const USERNAME = 'ucaw_user';
+
+type WhoAmI = { threadId: number; username: string | null; superUser: boolean };
+
+suite(
+	'user and role changes hold on every worker at acknowledgement',
+	{ skip: NO_FULL_WORKER_COVERAGE },
+	(ctx: ContextWithHarper) => {
+		let client: any;
+		let httpURL: string;
+		let password = 'ucaw-password-1';
+
+		const basic = (secret: string) => 'Basic ' + Buffer.from(`${USERNAME}:${secret}`).toString('base64');
+
+		async function whoAmI(secret: string): Promise<{ status: number; body?: WhoAmI }> {
+			const response = await fetchOnNewConnection(`${httpURL}/WhoAmI/`, { headers: { Authorization: basic(secret) } });
+			if (response.status !== 200) {
+				await response.text().catch(() => undefined);
+				return { status: response.status };
+			}
+			return { status: 200, body: (await response.json()) as WhoAmI };
+		}
+
+		function onEveryWorker(secret: string): Promise<WhoAmI[]> {
+			return observeEveryWorker(
+				async () => {
+					const { status, body } = await whoAmI(secret);
+					strictEqual(status, 200, `expected ${USERNAME} to authenticate`);
+					return body!;
+				},
+				(body) => body.threadId,
+				{ workerCount: WORKER_COUNT }
+			);
+		}
+
+		async function rejectedEverywhere(secret: string): Promise<void> {
+			const responses = await Promise.all(Array.from({ length: WORKER_COUNT * 8 }, () => whoAmI(secret)));
+			for (const { status, body } of responses) {
+				strictEqual(status, 401, `worker ${body?.threadId} still accepted the credential`);
+			}
+		}
+
+		before(async () => {
+			await setupHarperWithFixture(ctx, FIXTURE_PATH, { config: { threads: { count: WORKER_COUNT } } });
+			client = createApiClient(ctx.harper);
+			httpURL = ctx.harper.httpURL;
+			await assertEveryWorkerStarted(ctx);
+			await client
+				.req()
+				.send({ operation: 'add_role', role: ROLE, permission: { super_user: true } })
+				.expect(200);
+			await client
+				.req()
+				.send({ operation: 'add_user', role: ROLE, username: USERNAME, password, active: true })
+				.expect(200);
+			const deadline = Date.now() + 60_000;
+			while ((await whoAmI(password)).status !== 200) {
+				ok(Date.now() < deadline, 'the WhoAmI resource never became reachable');
+				await sleep(250);
+			}
+		});
+
+		after(async () => {
+			await teardownHarper(ctx);
+		});
+
+		test('a role permission change', async () => {
+			for (const body of await onEveryWorker(password)) strictEqual(body.superUser, true);
+			const roles = await client.req().send({ operation: 'list_roles' }).expect(200);
+			const role = roles.body.find((candidate: any) => candidate.role === ROLE);
+			await client
+				.req()
+				.send({ operation: 'alter_role', id: role.id, role: ROLE, permission: { super_user: false } })
+				.expect(200);
+			for (const body of await onEveryWorker(password)) {
+				strictEqual(body.superUser, false, `worker ${body.threadId} served the role as it was before alter_role`);
+			}
+		});
+
+		test('a password change', async () => {
+			await onEveryWorker(password);
+			const previous = password;
+			password = 'ucaw-password-2';
+			await client.req().send({ operation: 'alter_user', username: USERNAME, password }).expect(200);
+			await rejectedEverywhere(previous);
+			await onEveryWorker(password);
+		});
+
+		test('a deactivation', async () => {
+			await onEveryWorker(password);
+			await client.req().send({ operation: 'alter_user', username: USERNAME, active: false }).expect(200);
+			await rejectedEverywhere(password);
+		});
+	}
+);

resources/Resource.ts

@@ -1054,7 +1054,7 @@ function registerLiveSubscriptionForContext(subscription: any, resource: any, ad
 				// created later). Expiry (authExpiresAt above) is its only revocation.
 				fresh = user;
 			} else {
-				// Re-fetch current user state — the user/role cache is rebuilt on mutations — so a dropped or
+				// Re-read current user state from hdb_user/hdb_role, so a dropped or
 				// role-stripped user no longer authorizes.
 				const { findAndValidateUser } = require('../security/user');
 				fresh = await findAndValidateUser(username, undefined, false);

resources/Table.ts

@@ -4,13 +4,7 @@
  * table-level interactions, loading records, updating records, querying, and more.
  */
 
-import {
-	CONFIG_PARAMS,
-	OPERATIONS_ENUM,
-	SYSTEM_TABLE_NAMES,
-	SYSTEM_SCHEMA_NAME,
-	MAX_SET_TIMEOUT_MS,
-} from '../utility/hdbTerms.ts';
+import { CONFIG_PARAMS, OPERATIONS_ENUM, MAX_SET_TIMEOUT_MS } from '../utility/hdbTerms.ts';
 import { type Database } from 'lmdb';
 import { Script } from 'node:vm';
 import { randomUUID } from 'node:crypto';
@@ -69,7 +63,7 @@ import {
 	type ValidationIssue,
 } from '../utility/errors/hdbError.ts';
 import * as signalling from '../utility/signalling.ts';
-import { SchemaEventMsg, UserEventMsg } from '../server/threads/itc.js';
+import { SchemaEventMsg } from '../server/threads/itc.js';
 import {
 	databases,
 	table,
@@ -1070,7 +1064,6 @@ export function makeTable(options) {
 			// as they come in, and directly writing them to this table. We use the notification option to ensure
 			// that we don't re-broadcast these as "requested" changes back to the source.
 			(async () => {
-				let userRoleUpdate = false;
 				let lastSequenceId;
 				let pendingApplyFailures: Promise<void> | undefined;
 				const reportDroppedWrite = (event, context, error) => {
@@ -1133,12 +1126,6 @@ export function makeTable(options) {
 					if (isLockControlType(event.type)) return applyLockControlEvent(event, context);
 					const value = event.value;
 					const Table = event.table ? databases[databaseName][event.table] : TableResource;
-					if (
-						databaseName === SYSTEM_SCHEMA_NAME &&
-						(event.table === SYSTEM_TABLE_NAMES.ROLE_TABLE_NAME || event.table === SYSTEM_TABLE_NAMES.USER_TABLE_NAME)
-					) {
-						userRoleUpdate = true;
-					}
 					if (event.id === undefined) {
 						event.id = value[Table.primaryKey];
 						if (event.id === undefined) throw new Error('Replication message without an id ' + JSON.stringify(event));
@@ -1484,11 +1471,6 @@ export function makeTable(options) {
 									}
 								});
 								if (txnInProgress) txnInProgress.committed = commitResolution;
-								if (userRoleUpdate && commitResolution && !(commitResolution as any).waitingForUserChange) {
-									// if the user role changed, asynchronously signal the user change (but don't block this function)
-									commitResolution.then(() => signalling.signalUserChange(new UserEventMsg(process.pid)));
-									(commitResolution as any).waitingForUserChange = true; // only need to send one signal per transaction
-								}
 
 								if (event.onCommit) {
 									if (txnInProgress) {

security/DESIGN.md

@@ -173,3 +173,13 @@ Two consequences that are easy to miss:
   `settleDeferredCredentialRejection` _before_ they read `request.user`, so once a credential is deferred,
   resolving a principal from a different credential is a contradiction. Hence a rejected certificate
   identity stops resolution outright instead of falling through to Basic, the session, or the local bypass.
+
+## User and role lookups read the records, and nothing derived from them outlives them (`security/user.ts`)
+
+There is no per-thread copy of `hdb_user`/`hdb_role`: every lookup point-reads the user by name and its role by id through the primary store's record cache, so no writer (an operation, a replicated commit) has to announce a change for lookups to see it. Three rules keep that true:
+
+- **One committed state.** RocksDB point reads share no snapshot, so `readUserEntries` re-checks the user's version after reading its role and retries if it moved; otherwise one transaction that moves a user to role B and grants role A super_user could be read as the user on A with A's new grant.
+- **Derived data is keyed by entry version, not object identity.** With `storage.caching: false` every read returns a new object. The per-role memo of `appendSystemTablesToRole` + expanded operations, and `isCurrentUser`, compare versions; a `VERSION_REUSED` entry is compared by value instead, since its version no longer identifies one value. `auth.ts` runs `isCurrentUser` on every `authorizationCache` hit, so a cached principal is re-verified once its user or role record changes; a component's `server.getUser` principal is checked against the versions read for its name before it was resolved (`trackUserRecords`).
+- **Notifications are only for holders of a user.** `onUserChange` feeds live-subscription revocation and MCP list-changed from per-thread `hdb_user`/`hdb_role` subscriptions. It never subscribes to an unaudited table, because `subscribe()` would enable and persist auditing on it; on such a node those consumers fall back to their own backstops.
+
+LMDB lookups use the thread's shared read txn, which lmdb-js renews at most once per event-loop turn, so a commit on another thread is seen from the next turn. Resetting it per lookup would give each in-flight transaction its own reader slot. Enforced by `unitTests/security/userRecordLookups.test.js` and `unitTests/resources/replicatedUserWrites.test.js`.

security/auth.ts

@@ -1,4 +1,4 @@
-import { getSuperUser } from './user.ts';
+import { getSuperUser, isCurrentUser, trackUserRecords, userRecordVersions } from './user.ts';
 import { server } from '../server/Server.ts';
 import { resources } from '../resources/Resources.ts';
 import { validateOperationToken, validateRefreshToken, validateLoginToken, decodeJWT } from './tokenAuthentication.ts';
@@ -8,8 +8,6 @@ import * as env from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS, AUTH_AUDIT_STATUS, AUTH_AUDIT_TYPES } from '../utility/hdbTerms.ts';
 import harperLogger from '../utility/logging/harper_logger.ts';
 const { forComponent, AuthAuditLog, errorForLog } = harperLogger;
-import serverHandlers from '../server/itc/serverHandlers.js';
-const { user } = serverHandlers;
 import { Headers, addVaryHeader, SHARED_CACHE_OPTIN, PRIVATE_SCOPE } from '../server/serverHelpers/Headers.ts';
 import { convertToMS } from '../utility/common_utils.ts';
 import { verifyCertificate } from './certificateVerification/index.ts';
@@ -231,7 +229,10 @@ export async function authentication(request, nextHandler) {
 			let cachedUser = authorizationCache.get(authorization);
 			// A cached Bearer identity must not outlive its token: expiry is the only revocation
 			// mechanism for scoped tokens, so it has to be exact, not cache-TTL-fuzzy.
-			if (cachedUser?.authExpiresAt && cachedUser.authExpiresAt * 1000 <= Date.now()) {
+			if (
+				cachedUser &&
+				((cachedUser.authExpiresAt && cachedUser.authExpiresAt * 1000 <= Date.now()) || !isCurrentUser(cachedUser))
+			) {
 				authorizationCache.delete(authorization);
 				cachedUser = undefined;
 			}
@@ -257,7 +258,12 @@ export async function authentication(request, nextHandler) {
 							username = decoded.slice(0, colonIndex);
 							password = decoded.slice(colonIndex + 1);
 							// legacy support for passing in blank username and password to indicate no auth
-							newUser = username || password ? await server.getUser(username, password, request) : null;
+							if (username || password) {
+								// read first: a component's server.getUser principal carries no record versions itself
+								const versions = userRecordVersions(username);
+								newUser = await server.getUser(username, password, request);
+								trackUserRecords(newUser, versions);
+							} else newUser = null;
 							break;
 						case 'Bearer':
 							try {
@@ -483,9 +489,6 @@ export async function authentication(request, nextHandler) {
 setInterval(() => {
 	authorizationCache = new Map();
 }, env.get(CONFIG_PARAMS.AUTHENTICATION_CACHETTL)).unref();
-user.addListener(() => {
-	authorizationCache = new Map();
-});
 let started = false;
 export function handleApplication(scope: import('../components/Scope.ts').Scope) {
 	if (started) return;

security/authn/oidc/tokenExchange.ts

@@ -17,7 +17,7 @@ import { loggerWithTag } from '../../../utility/logging/logger.ts';
 import harperLogger from '../../../utility/logging/harper_logger.ts';
 import * as env from '../../../utility/environment/environmentManager.ts';
 import { AUTH_AUDIT_STATUS, AUTH_AUDIT_TYPES, CONFIG_PARAMS } from '../../../utility/hdbTerms.ts';
-import { getUsersWithRolesCache } from '../../user.ts';
+import { getUserWithRole } from '../../user.ts';
 import { createOperationToken } from '../../tokenAuthentication.ts';
 import { rejectToken, verifyIdentityToken } from './identityToken.ts';
 import { matchTrustPolicyClaims } from './claims.ts';
@@ -264,8 +264,7 @@ export async function exchangeOidcToken(req: any) {
 
 		// Resolved before the token is spent, so a policy naming a deleted or deactivated user fails
 		// without burning a token the runner cannot re-mint.
-		const users = await getUsersWithRolesCache();
-		const user = users?.get(policy.user);
+		const user = getUserWithRole(policy.user);
 		if (!user) rejectToken(`trust policy '${policy.id}' names user '${policy.user}', which does not exist`);
 		if (user.active === false) rejectToken(`trust policy '${policy.id}' names inactive user '${policy.user}'`);
 		username = user.username;