test(server): promote validated QA regression anchors (static, deploy, shutdown-drain, secrets)

integrationTests/components/fixtures/secret-audit-leak/config.yaml

@@ -0,0 +1,7 @@
+# QA-517 — hdb_secret audit-log / generic-table plaintext leak probe.
+# hdb_secret is a built-in system table (no app schema needed). registerCustody.js is wired in via
+# HARPER_BUILTIN_COMPONENTS (see the test file), but that only registers the module under
+# TRUSTED_RESOURCE_PLUGINS — it must ALSO appear as a top-level component key here (same pattern
+# as qa487's qa487RegisterDecryptor) for componentLoader to actually process/load it.
+rest: true
+qa517RegisterCustody: true

integrationTests/components/fixtures/secret-audit-leak/registerCustody.js

@@ -0,0 +1,46 @@
+// QA-517 test-only builtin component.
+//
+// components/secretOperations.ts requires `getSecretCustody()` to be non-null for `set_secret`
+// with a plaintext `value` to work at all ("secrets custody is not initialized on this node").
+// Core ships no custody (that's the Harper Pro secrets component's job); this registers a real,
+// in-process RSA keypair as custody so set_secret's real encrypt-then-put path runs end-to-end
+// (no mocks), exactly like the shipped unit test's `installCustody()` helper.
+//
+// secretOperations.ts's handlers dispatch on the ops-API MAIN THREAD ONLY (see its own comment),
+// so registration must happen on the main thread too — the `handleApplication` Plugin API (used
+// by qa487's registerDecryptor.js) only runs per-WORKER (componentLoader.ts's `resources.isWorker`
+// gate), which would be invisible to the main-thread dispatch. `startOnMainThread` (old Extension
+// API, still supported — componentLoader.ts only warns, doesn't block) runs exactly once on the
+// main thread, which is what's needed here.
+import { generateKeyPairSync } from 'node:crypto';
+import { join, dirname } from 'node:path';
+import { pathToFileURL, fileURLToPath } from 'node:url';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export async function startOnMainThread() {
+	// Reach the running install's own dist build directly, so this external fixture file shares
+	// the exact module instance (and registry state) that components/secretOperations.ts's dist
+	// counterpart imports.
+	const distSecretDecryptorPath = join(__dirname, '..', '..', '..', '..', 'dist', 'resources', 'secretDecryptor.js');
+	const { registerSecretCustody } = await import(pathToFileURL(distSecretDecryptorPath).toString());
+
+	const { publicKey, privateKey } = generateKeyPairSync('rsa', {
+		modulusLength: 2048,
+		publicKeyEncoding: { type: 'spki', format: 'pem' },
+		privateKeyEncoding: { type: 'pkcs8', format: 'pem' },
+	});
+
+	const distSecretEnvelopePath = join(__dirname, '..', '..', '..', '..', 'dist', 'utility', 'secretEnvelope.js');
+	const { fingerprintOf, decryptEnvelope } = await import(pathToFileURL(distSecretEnvelopePath).toString());
+	const fingerprint = fingerprintOf(publicKey);
+	const PREFIX = 'enc:v1:';
+
+	registerSecretCustody({
+		decrypt: (value) => decryptEnvelope(value.slice(PREFIX.length), privateKey, fingerprint),
+		getPublicKey: () => ({ publicKey, fingerprint }),
+	});
+
+	// eslint-disable-next-line no-console
+	console.log(`QA517 fake secret custody registered on main thread, fingerprint=${fingerprint}`);
+}

integrationTests/components/fixtures/shutdown-drain-e2e/config.yaml

@@ -0,0 +1,3 @@
+jsResource:
+  files: resources.js
+rest: true

integrationTests/components/fixtures/shutdown-drain-e2e/resources.js

@@ -0,0 +1,94 @@
+// QA-519 — end-to-end verification of the shutdown-drain mechanism
+// (components/shutdownDrain.ts, commit 9018760e4, PR #1621).
+//
+// The shipped unit tests (unitTests/components/shutdownDrain.test.js) only exercise the pure
+// functions with fake drain objects — never a real worker, a real SHUTDOWN, or real in-flight
+// work. This fixture registers a REAL ShutdownDrain inside an actual HTTP worker thread and
+// gives the test a way to observe, via a marker file, exactly when the worker's simulated
+// in-flight task finished relative to when the worker itself exited.
+//
+// To register into the SAME per-worker `drains` registry that
+// `server/threads/threadServer.js` reads from (module state there is a plain in-memory `Set`,
+// naturally scoped per worker thread — see the doc comment on shutdownDrain.ts), this file must
+// land in Node's REAL native CJS `require` cache, not Harper's sandboxed VM module loader's
+// private per-component cache (which would produce an isolated copy of the module with its own
+// empty `Set`). `createRequire(import.meta.url)` escapes the sandbox for exactly this file,
+// giving a genuine native `require` bound to this file's real location; requiring the absolute
+// path to the compiled `dist/components/shutdownDrain.js` through it resolves to the exact same
+// cache entry `threadServer.js` itself populated at boot.
+import { createRequire } from 'node:module';
+import { threadId } from 'node:worker_threads';
+import { appendFileSync } from 'node:fs';
+
+const nativeRequire = createRequire(import.meta.url);
+const { registerShutdownDrain } = nativeRequire(process.env.QA519_SHUTDOWN_DRAIN_ABS_PATH);
+
+const MARKER_FILE = process.env.QA519_MARKER_FILE;
+const parsedTaskDelay = Number(process.env.QA519_TASK_DELAY_MS);
+const TASK_DELAY_MS = Number.isInteger(parsedTaskDelay) && parsedTaskDelay > 0 ? parsedTaskDelay : 2500;
+
+function log(tag) {
+	try {
+		appendFileSync(MARKER_FILE, `${tag} t=${Date.now()} pid=${process.pid} tid=${threadId}\n`);
+	} catch {
+		// best effort — a lost log line just weakens the test's evidence, not worth crashing over
+	}
+}
+
+let taskDone = false;
+let stallMode = false;
+let taskPromise = null;
+
+function startTask(stall) {
+	taskDone = false;
+	stallMode = !!stall;
+	log(stall ? 'B_START' : 'A_START');
+	if (stall) {
+		// Simulates a hung/stuck operation: never resolves on its own. hasWork() keeps reporting
+		// true forever, so the ONLY way this worker can exit is the drain ceiling's force-kill
+		// (runShutdownDrains' own deadline race abandoning this drain), not this promise settling.
+		taskPromise = new Promise(() => {});
+	} else {
+		taskPromise = new Promise((resolve) => {
+			setTimeout(() => {
+				taskDone = true;
+				log('A_DONE');
+				resolve();
+			}, TASK_DELAY_MS);
+		});
+	}
+}
+
+registerShutdownDrain({
+	hasWork() {
+		return stallMode ? true : taskPromise !== null && !taskDone;
+	},
+	async drain() {
+		log(stallMode ? 'B_DRAIN_ENTER' : 'A_DRAIN_ENTER');
+		await taskPromise;
+		// Only reached if taskPromise actually settles before runShutdownDrains' own deadline
+		// race abandons it — never true in stall mode (taskPromise there never resolves).
+		log(stallMode ? 'B_DRAIN_EXIT_UNEXPECTED' : 'A_DRAIN_EXIT');
+	},
+});
+
+// Fires synchronously as part of realExit()/process.exit() — the last thing this worker does.
+process.on('exit', () => {
+	log(stallMode ? 'B_EXIT' : 'A_EXIT');
+});
+
+export class TaskProbe extends Resource {
+	static loadAsInstance = false;
+	async get(query) {
+		const action = query && query.get ? query.get('action') : undefined;
+		if (action === 'start') {
+			startTask(false);
+			return { started: true, mode: 'normal', delayMs: TASK_DELAY_MS, pid: process.pid, tid: threadId };
+		}
+		if (action === 'start-stall') {
+			startTask(true);
+			return { started: true, mode: 'stall', pid: process.pid, tid: threadId };
+		}
+		return { taskDone, stallMode, pid: process.pid, tid: threadId };
+	}
+}

integrationTests/components/secret-audit-leak.test.ts

@@ -0,0 +1,140 @@
+/**
+ * hdb_secret plaintext-leak probe — harper#715 / PR#1554.
+ *
+ * Pins: `set_secret` (create + rotate) never surfaces plaintext via `read_audit_log`
+ * (blocked 403) or via a generic system-table read (search_by_value / sql only return
+ * the `enc:v1:` ciphertext envelope, never the raw value).
+ *
+ * Run: npm run test:integration -- "integrationTests/components/secret-audit-leak.test.ts"
+ */
+import { suite, test, before, after } from 'node:test';
+import { ok, equal } from 'node:assert';
+import { resolve } from 'node:path';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { setupHarperWithFixture, teardownHarper, type ContextWithHarper } from '@harperfast/integration-testing';
+// @ts-expect-error utils/client.mjs has no type declarations; runtime resolves fine
+import { createApiClient } from '../apiTests/utils/client.mjs';
+
+const FIXTURE_PATH = resolve(import.meta.dirname, 'fixtures/secret-audit-leak');
+const skipSuite = process.platform === 'win32';
+
+const SECRET_NAME = 'qa_test_secret';
+const MARKER_A = 'qa517-super-secret-plaintext-12345-AAA';
+const MARKER_B = 'qa517-super-secret-plaintext-67890-BBB-rotated';
+
+suite(
+	'hdb_secret audit-log / generic-table plaintext leak probe (#715)',
+	{ skip: skipSuite },
+	(ctx: ContextWithHarper) => {
+		let client: ReturnType<typeof createApiClient>;
+
+		before(async () => {
+			await setupHarperWithFixture(ctx, FIXTURE_PATH, {
+				config: { logging: { auditLog: true } },
+				env: {
+					HARPER_BUILTIN_COMPONENTS:
+						'qa517RegisterCustody=@/integrationTests/components/fixtures/secret-audit-leak/registerCustody.js',
+				},
+			});
+			client = createApiClient(ctx.harper);
+
+			const deadline = Date.now() + 30_000;
+			while (Date.now() < deadline) {
+				const r = await client.req().send({ operation: 'get_secrets_public_key' });
+				if (r.status === 200 && r.body?.fingerprint) break;
+				await sleep(250);
+			}
+		});
+
+		after(async () => {
+			await teardownHarper(ctx);
+		});
+
+		test('system.hdb_secret exists as a built-in system table', async () => {
+			const r = await client.req().send({ operation: 'describe_table', database: 'system', table: 'hdb_secret' });
+			equal(r.status, 200, `describe_table failed: ${JSON.stringify(r.body)}`);
+		});
+
+		test('set_secret encrypts on ingest; read_audit_log on system.hdb_secret is blocked with 403', async () => {
+			const setResp = await client.req().send({ operation: 'set_secret', name: SECRET_NAME, value: MARKER_A });
+			equal(setResp.status, 200, `set_secret failed: ${JSON.stringify(setResp.body)}`);
+			ok(!JSON.stringify(setResp.body).includes(MARKER_A), 'set_secret response must not echo the plaintext');
+			equal(setResp.body.created, true);
+
+			const auditResp = await client.req().send({
+				operation: 'read_audit_log',
+				database: 'system',
+				table: 'hdb_secret',
+				search_type: 'hash_value',
+				search_values: [SECRET_NAME],
+			});
+
+			if (auditResp.status !== 403) {
+				const bodyText = JSON.stringify(auditResp.body);
+				ok(!bodyText.includes(MARKER_A), 'read_audit_log body must never contain the plaintext value');
+			}
+			equal(auditResp.status, 403, 'read_audit_log on system.hdb_secret must be blocked by design');
+		});
+
+		test('rotation (update) — read_audit_log still blocked; no intermediate plaintext observable', async () => {
+			const setResp = await client.req().send({ operation: 'set_secret', name: SECRET_NAME, value: MARKER_B });
+			equal(setResp.status, 200, `set_secret (rotate) failed: ${JSON.stringify(setResp.body)}`);
+			equal(setResp.body.created, false, 'rotation should report created:false (update path)');
+			ok(
+				!JSON.stringify(setResp.body).includes(MARKER_B),
+				'rotated set_secret response must not echo the new plaintext'
+			);
+
+			const auditResp = await client.req().send({
+				operation: 'read_audit_log',
+				database: 'system',
+				table: 'hdb_secret',
+				search_type: 'hash_value',
+				search_values: [SECRET_NAME],
+			});
+
+			if (auditResp.status !== 403) {
+				const bodyText = JSON.stringify(auditResp.body);
+				ok(
+					!bodyText.includes(MARKER_B) && !bodyText.includes(MARKER_A),
+					'read_audit_log body must never contain either plaintext value'
+				);
+			}
+			equal(auditResp.status, 403, 'read_audit_log on system.hdb_secret must be blocked by design (rotation case)');
+		});
+
+		test('generic system-table read (search_by_value, sql) shows only the enc:v1: envelope, never plaintext', async () => {
+			const r = await client.req().send({
+				operation: 'search_by_value',
+				database: 'system',
+				table: 'hdb_secret',
+				search_attribute: 'name',
+				search_value: SECRET_NAME,
+				get_attributes: ['*'],
+			});
+			equal(r.status, 200, `search_by_value failed: ${JSON.stringify(r.body)}`);
+			equal(r.body.length, 1, 'expected exactly one row for the test secret');
+			const row = r.body[0];
+
+			ok(
+				typeof row.envelope === 'string' && row.envelope.startsWith('enc:v1:'),
+				'envelope must be the enc:v1: ciphertext form'
+			);
+			const rowText = JSON.stringify(row);
+			ok(
+				!rowText.includes(MARKER_A) && !rowText.includes(MARKER_B),
+				'generic table read must never expose plaintext values'
+			);
+
+			const sqlResp = await client
+				.req()
+				.send({ operation: 'sql', sql: `SELECT * FROM system.hdb_secret WHERE name = '${SECRET_NAME}'` });
+			equal(sqlResp.status, 200, `sql failed: ${JSON.stringify(sqlResp.body)}`);
+			const sqlText = JSON.stringify(sqlResp.body);
+			ok(
+				!sqlText.includes(MARKER_A) && !sqlText.includes(MARKER_B),
+				'SQL generic read must never expose plaintext values'
+			);
+		});
+	}
+);