cherry-pick: Preserve 5.2 rollback compatibility for first-time table creates (conflicts → v5.3)

DESIGN.md

@@ -45,7 +45,7 @@ Index of the design notes for the harper core: one line per note, grouped by the
 - [Subscription version selection (`Table.ts`)](resources/DESIGN.md#subscription-version-selection-tablets) — Replay and live delivery share primary-version filtering, reading the current entry once per audit record rather than per subscriber; messages remain independent and raw events preserve history by default.
 - [Audit-store `'committed'` notification batching (`transactionBroadcast.ts`)](resources/DESIGN.md#audit-store-committed-notification-batching-transactionbroadcastts) — `'committed'` notifications are deferred with `setImmediate`, collapsed per turn and batched with yields; the ancestor key walk must strictly shrink; a subscription ended mid-delivery leaves its key's array to the loop walking it, and every delivery loop is bounded by what it started with.
 - [`createBlob(readable)` and `table.put()` don't synchronously drain the source](resources/DESIGN.md#createblobreadable-and-tableput-dont-synchronously-drain-the-source) — `table.put()` returns before a `Readable`-backed blob has drained; finalize a hash on the stream's end or await `storageInfo.saving`.
-- [Table drops, the `dropping` tombstone, and ghost tables](resources/DESIGN.md#table-drops-the-dropping-tombstone-and-ghost-tables) — `dropTable()` persists a `dropping` tombstone before destructive work; boot and a same-name create complete an interrupted drop.
+- [Table drops, the `dropping` tombstone, and ghost tables](resources/DESIGN.md#table-drops-the-dropping-tombstone-and-ghost-tables) — `dropTable()` persists a `dropping` tombstone before destructive work; boot and a same-name create complete an interrupted drop. First-time creates keep legacy names for rollback; recovery preserves names owned by a live table (harper#3102).
 - [The exclusive `update-attributes` lock is a bounded synchronous wait, and drop-then-recreate needs the column-family eviction fix (`Table.ts`)](resources/DESIGN.md#the-exclusive-update-attributes-lock-is-a-bounded-synchronous-wait-and-drop-then-recreate-needs-the-column-family-eviction-fix-tablets) — The lock is a bounded `Atomics.wait` with structural release, acquired before any live-`Table` mutation; drop-then-recreate needs the `@harperfast/rocksdb-js` column-family eviction fix.
 - [RocksDB transaction log purges are database-wide only (`ResourceBridge.deleteTransactionLogsBefore`)](resources/DESIGN.md#rocksdb-transaction-log-purges-are-database-wide-only-resourcebridgedeletetransactionlogsbefore) — RocksDB transaction logs are per database, so a table-scoped `delete_transaction_logs_before` is rejected with 400.
 - [Boot replay reads bypass the primary-record cache](resources/DESIGN.md#boot-replay-reads-bypass-the-primary-record-cache) — Snapshot reads avoid retaining the durable backlog through WeakRefs during synchronous boot replay.

integrationTests/upgrade/first-create-downgrade.test.ts

@@ -0,0 +1,136 @@
+/**
+ * Tables first created after a minor upgrade remain readable and writable by 5.2,
+ * including secondary indexes and writes carried through a re-upgrade (#3102).
+ */
+import { suite, test, after } from 'node:test';
+import assert from 'node:assert';
+import { delimiter, join } from 'node:path';
+import { readFileSync } from 'node:fs';
+import {
+	startHarper,
+	killHarper,
+	teardownHarper,
+	sendOperation,
+	type ContextWithHarper,
+} from '@harperfast/integration-testing';
+
+const previousPath = process.env.HARPER_PREVIOUS_MINOR_PATH?.split(delimiter).find((path) => {
+	const { version } = JSON.parse(readFileSync(join(path, 'package.json'), 'utf8'));
+	return version.startsWith('5.2.');
+});
+
+suite(
+	'first-time table create survives 5.2 rollback',
+	{
+		skip:
+			!previousPath ||
+			process.env.HARPER_RUNTIME === 'bun' ||
+			process.env.HARPER_STORAGE_ENGINE === 'lmdb' ||
+			process.platform === 'win32',
+		timeout: 300_000,
+	},
+	(ctx: ContextWithHarper) => {
+		after(async () => teardownHarper(ctx));
+
+		test('preserves primary rows, secondary indexes and rollback writes', async () => {
+			const previousBin = join(previousPath!, 'dist', 'bin', 'harper.js');
+			await startHarper(ctx, { config: {}, env: { TC_AGREEMENT: 'yes' }, harperBinPath: previousBin });
+			await killHarper(ctx);
+			await startHarper(ctx, { config: {} });
+			const schemas = [undefined, 'new_on_current'];
+			await sendOperation(ctx.harper, { operation: 'create_schema', schema: 'new_on_current' });
+			const records = [
+				{ id: 'a', category: 'current' },
+				{ id: 'b', category: 'current' },
+			];
+			for (const schema of schemas) {
+				await sendOperation(ctx.harper, {
+					operation: 'create_table',
+					schema,
+					table: 'born_on_current',
+					primary_key: 'id',
+					attributes: [
+						{ name: 'id', type: 'ID' },
+						{ name: 'category', type: 'String', indexed: true },
+					],
+				});
+				await sendOperation(ctx.harper, { operation: 'upsert', schema, table: 'born_on_current', records });
+			}
+			await sendOperation(ctx.harper, { operation: 'create_table', table: 'recreated', primary_key: 'id' });
+			await sendOperation(ctx.harper, { operation: 'upsert', table: 'recreated', records: [{ id: 'retired' }] });
+			await sendOperation(ctx.harper, { operation: 'drop_table', table: 'recreated' });
+			await sendOperation(ctx.harper, { operation: 'create_table', table: 'recreated', primary_key: 'id' });
+			await sendOperation(ctx.harper, { operation: 'upsert', table: 'recreated', records: [{ id: 'stamped' }] });
+			await sendOperation(ctx.harper, { operation: 'create_table', table: 'rollback_recreate', primary_key: 'id' });
+			await sendOperation(ctx.harper, {
+				operation: 'upsert',
+				table: 'rollback_recreate',
+				records: [{ id: 'retired' }],
+			});
+			await sendOperation(ctx.harper, { operation: 'drop_table', table: 'rollback_recreate' });
+			await killHarper(ctx);
+			await startHarper(ctx, { config: {}, env: { CONFIRM_DOWNGRADE: 'yes' }, harperBinPath: previousBin });
+			await sendOperation(ctx.harper, { operation: 'create_table', table: 'rollback_recreate', primary_key: 'id' });
+			await sendOperation(ctx.harper, {
+				operation: 'upsert',
+				table: 'rollback_recreate',
+				records: [{ id: 'legacy-recreated' }],
+			});
+
+			const read = (schema: string | undefined, attribute = 'id', value = '*') =>
+				sendOperation(ctx.harper, {
+					operation: 'search_by_value',
+					schema,
+					table: 'born_on_current',
+					search_attribute: attribute,
+					search_value: value,
+					get_attributes: ['id', 'category'],
+				});
+			assert.deepStrictEqual(
+				await sendOperation(ctx.harper, {
+					operation: 'search_by_value',
+					table: 'recreated',
+					search_attribute: 'id',
+					search_value: '*',
+					get_attributes: ['*'],
+				}),
+				[],
+				'5.2 does not support generation-stamped recreates'
+			);
+			const rollbackRecord = { id: 'c', category: 'rollback' };
+			for (const schema of schemas) {
+				assert.deepStrictEqual(await read(schema), records);
+				assert.deepStrictEqual(await read(schema, 'category', 'current'), records);
+				await sendOperation(ctx.harper, {
+					operation: 'upsert',
+					schema,
+					table: 'born_on_current',
+					records: [rollbackRecord],
+				});
+			}
+			await killHarper(ctx);
+			await startHarper(ctx, { config: {} });
+			const readRecreated = () =>
+				sendOperation(ctx.harper, {
+					operation: 'search_by_value',
+					table: 'rollback_recreate',
+					search_attribute: 'id',
+					search_value: '*',
+					get_attributes: ['id'],
+				});
+			assert.deepStrictEqual(await readRecreated(), [{ id: 'legacy-recreated' }]);
+			await killHarper(ctx);
+			await startHarper(ctx, { config: {} });
+			assert.deepStrictEqual(
+				await readRecreated(),
+				[{ id: 'legacy-recreated' }],
+				'a retired journal must not reclaim the bare table recreated on 5.2'
+			);
+			for (const schema of schemas) {
+				assert.deepStrictEqual(await read(schema), [...records, rollbackRecord]);
+				assert.deepStrictEqual(await read(schema, 'category', 'current'), records);
+				assert.deepStrictEqual(await read(schema, 'category', 'rollback'), [rollbackRecord]);
+			}
+		});
+	}
+);

resources/DESIGN.md

@@ -617,6 +617,8 @@ interrupted drop instead of resurrecting the table. Without this, surviving cata
 silently re-opened with create-if-missing on the next start, which resurrects "deleted" tables
 (with their data, if the column families were never actually removed).
 
+**First-time creates and 5.2 rollback (harper#3102).** First-time RocksDB creates use bare primary/index names only when this node has no `/dropped/<table>` row, no journal row for the table, and no `T/` column family. Every RocksDB drop completion (`retireRocksStores` in `Table.ts`, `completeInterruptedDrop`) writes that untimed name-history row (`recordTableNameHistory`) before removing the tombstone, so any name history requires a generation stamp, including after reclamation. The load parser skips `/dropped/` rows; 5.4 reads the same row as its untimed drop marker (harper#2962), which a timed drop overwrites. Incoming primary-attribute generations are replaced by this node's physical naming choice. Create journals have their own identity; when a creating and retired journal name the same primary, the creating row leaves that primary to the retired row's blob sweep. Recovery reads their current phases under the catalog lock. Their `creatingStores` field must stay separate from `stores`, which shipped 5.3 readers destructively reclaim without recognizing a published bare primary. Reclamation always preserves stores owned by the live, non-dropping catalog and never sweeps that primary's blobs: a 5.2 writer can reuse bare names while ignoring the retired journal. Previously stamped tables retain their names and remain outside 5.2 rollback support, as do stamped recreates. During interrupted-drop recovery, a bare predecessor can remain until its asynchronous reclamation finishes; 5.2 ignores that recovery state and can expose predecessor rows through a recreate. `unitTests/resources/dropTableGeneration.test.js` enforces naming, crash recovery, and live-store ownership; `integrationTests/upgrade/first-create-downgrade.test.ts` exercises the real 5.2 round trip.
+
 ## The exclusive `update-attributes` lock is a bounded synchronous wait, and drop-then-recreate needs the column-family eviction fix (`Table.ts`)
 
 The create/schema-update path's exclusive `update-attributes` lock is a synchronous bounded wait

resources/Table.ts

@@ -88,6 +88,8 @@ import {
 	sweepDroppedTableBlobs,
 	storeNameFor,
 	storeNamesFor,
+	liveStoreNamesFor,
+	recordTableNameHistory,
 	isReadOnlyMode,
 } from './databases.ts';
 import { notifyReplicatedApplyFailure } from './replicatedApplyFailure.ts';
@@ -3131,6 +3133,13 @@ export function makeTable(options): TableResourceClass {
 					// has stopped and the wrapper has retired their storage.
 					if (!(await retireFullTextStorage())) return false;
 					const removed = withUpdateAttributesLock(rootStore, `table '${databaseName}.${tableName}'`, () => {
+						const currentPrimary = (dbisDb as any).getSync(primaryCatalogKey);
+						if (
+							!currentPrimary?.dropping ||
+							(currentPrimary.tableId != null && currentPrimary.tableId !== tableId) ||
+							currentPrimary.dropGeneration !== dropGeneration
+						)
+							return false;
 						const stores = storeNamesFor(dbisDb, tableName, generation);
 						const retiredStores = recordRetiredGeneration(
 							dbisDb,
@@ -3164,25 +3173,35 @@ export function makeTable(options): TableResourceClass {
 							if (!droppedStores.has(columnName) && (rootStore as any).columns.includes(columnName))
 								dropColumnFamily(rootStore, columnName);
 						}
-						const currentPrimary = (dbisDb as any).getSync(primaryCatalogKey);
+						const remainingPrimary = (dbisDb as any).getSync(primaryCatalogKey);
 						if (
-							!currentPrimary?.dropping ||
-							(currentPrimary.tableId != null && currentPrimary.tableId !== tableId) ||
-							currentPrimary.dropGeneration !== dropGeneration
+							!remainingPrimary?.dropping ||
+							(remainingPrimary.tableId != null && remainingPrimary.tableId !== tableId) ||
+							remainingPrimary.dropGeneration !== dropGeneration
 						)
 							return false;
 						for (const key of dbisDb.getKeys({ start: tableName + '/', end: tableName + '0' })) {
 							if (key !== primaryCatalogKey) dbisDb.remove(key);
 						}
+						recordTableNameHistory(dbisDb, tableName, currentPrimary.tableId);
 						dbisDb.remove(primaryCatalogKey);
 						return true;
 					});
-					if (removed) await dbisDb.committed;
+					if (!removed) {
+						withUpdateAttributesLock(rootStore, `retire stale stores of '${databaseName}.${tableName}'`, () => {
+							const owned = liveStoreNamesFor(dbisDb, tableName);
+							const columns = new Set<string>((rootStore as any).columns);
+							for (const store of [primaryStore, ...Object.values(indices)]) {
+								if (store.name && columns.has(store.name) && !owned.has(store.name))
+									dropColumnFamily(rootStore, store.name);
+							}
+						});
+					} else await dbisDb.committed;
 					const label = `${databaseName}.${tableName}`;
 					const settled = await settlePhysicalDrops(rootStore, label);
 					await sweepDroppedTableBlobs(primaryStore, label);
 					if (!settled) finishDroppedTableBlobSweep(rootStore, primaryStore, label);
-					return true;
+					return removed;
 				} finally {
 					releaseDropMark();
 				}