fix(autonumber): single owner = persistent driver sequence (#1603) (#1608)

Autonumber values were generated twice: the SQL driver's persistent,
atomic _objectstack_sequences table AND a non-persistent in-memory
counter in the engine. The engine pre-filled before calling the driver,
so the driver always saw a value and skipped — leaving the persistent
sequence as dead code and risking duplicate numbers across instances /
after a restart.

- spec: DriverCapabilities.autonumber (optional) capability flag.
- driver-sql: advertise supports.autonumber=true; bulkCreate now fills
  autonumber too (was create/upsert only); parse honors autonumberFormat
  OR format.
- objectql: engine defers to the driver when it advertises native
  autonumber support (no more pre-fill / in-memory shadow); keeps the
  in-memory fallback for memory/mongodb; fallback also reads either
  format key; record-validation exempts autonumber from the required
  check (runtime-owned, assigned post-validation).

Tests: driver bulkCreate + autonumberFormat + capability (driver-sql
140 pass); engine defer-vs-fallback + validator exemption (objectql
526 pass). Existing data respected via scanMaxNumericTail bootstrap.

Co-authored-by: Jack Zhuang <277994282+os-zhuang@users.noreply.github.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: 3 people

.changeset/autonumber-driver-consolidation.md

@@ -0,0 +1,37 @@
+---
+'@objectstack/spec': minor
+'@objectstack/objectql': minor
+'@objectstack/driver-sql': minor
+---
+
+fix(autonumber): one owner for autonumber generation — the persistent driver sequence (#1603)
+
+Autonumber values were generated in TWO places: the SQL driver's persistent,
+atomic `_objectstack_sequences` table AND a non-persistent in-memory counter in
+the ObjectQL engine. Because the engine pre-filled the field BEFORE calling the
+driver, the driver always saw a value already set and skipped — so the
+persistent sequence was effectively dead code, and a multi-instance / post-restart
+deployment could mint duplicate numbers from the in-memory counter.
+
+This makes generation single-owner:
+
+- **`@objectstack/spec`** — `DriverCapabilities` gains an optional `autonumber`
+  flag: "driver natively generates persistent autonumber/sequence values".
+
+- **`@objectstack/driver-sql`** — advertises `supports.autonumber = true`.
+  `bulkCreate()` now fills autonumber fields too (previously only `create()` /
+  `upsert()` did), so bulk inserts also draw from the persistent sequence.
+  Field parsing now honors either the spec-canonical `autonumberFormat` key OR
+  the `format` shorthand (both appear in metadata).
+
+- **`@objectstack/objectql`** — when the driver advertises native autonumber
+  support, the engine NO LONGER pre-fills (it defers entirely to the persistent
+  driver sequence as the single source of truth). For drivers without native
+  support (memory, mongodb) the in-memory fallback is unchanged. The fallback
+  also now reads either `autonumberFormat` or `format`. Record-validation
+  exempts `autonumber` fields from the `required` check — the value is
+  runtime-owned and assigned after validation, so a required record number is
+  never rejected as "missing".
+
+No metadata changes required. Existing data is respected: the driver bootstraps
+each sequence from the current max numeric tail on first use.

packages/objectql/src/engine-autonumber-defer.test.ts

@@ -0,0 +1,115 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { ObjectQL } from './engine';
+import { SchemaRegistry } from './registry';
+import type { IDataDriver } from '@objectstack/spec/contracts';
+
+/**
+ * #1603 — autonumber generation is owned by ONE layer.
+ *
+ * When the driver advertises `supports.autonumber === true` (the SQL driver,
+ * which has a persistent `_objectstack_sequences` table), the engine must NOT
+ * pre-fill `autonumber` fields with its in-memory counter — it defers to the
+ * driver so the persistent sequence is the single source of truth. When the
+ * driver does NOT advertise it (memory / mongodb), the engine keeps its
+ * in-memory fallback so nothing regresses.
+ *
+ * A `required` autonumber must pass insert-validation in BOTH cases (the value
+ * is runtime-owned, assigned after validation in the native-driver case).
+ */
+vi.mock('./registry', () => {
+  const instance: any = {
+    getObject: vi.fn(),
+    resolveObject: vi.fn((n: string) => instance.getObject(n)),
+    registerObject: vi.fn(),
+    getObjectOwner: vi.fn(),
+    registerNamespace: vi.fn(),
+    registerKind: vi.fn(),
+    registerItem: vi.fn(),
+    registerApp: vi.fn(),
+    installPackage: vi.fn(),
+    reset: vi.fn(),
+    metadata: { get: vi.fn(() => new Map()) },
+  };
+  function SchemaRegistry() {
+    return instance;
+  }
+  Object.assign(SchemaRegistry, instance);
+  return {
+    SchemaRegistry,
+    computeFQN: (_ns: string | undefined, name: string) => name,
+    parseFQN: (fqn: string) => ({ namespace: undefined, shortName: fqn }),
+    RESERVED_NAMESPACES: new Set(['base', 'system']),
+  };
+});
+
+function makeDriver(supportsAutonumber: boolean): IDataDriver & { created: any[] } {
+  const created: any[] = [];
+  const driver: any = {
+    name: supportsAutonumber ? 'sql' : 'memory',
+    supports: supportsAutonumber ? { autonumber: true } : {},
+    connect: vi.fn().mockResolvedValue(undefined),
+    disconnect: vi.fn().mockResolvedValue(undefined),
+    // seedAutonumber (fallback path) reads existing rows; none exist yet.
+    find: vi.fn().mockResolvedValue([]),
+    findOne: vi.fn(),
+    create: vi.fn(async (_obj: string, row: any) => {
+      created.push(row);
+      // Native driver assigns the autonumber itself (post-validation).
+      return supportsAutonumber ? { id: 'r1', ...row, doc_no: 'D-0001' } : { id: 'r1', ...row };
+    }),
+    update: vi.fn(),
+    delete: vi.fn(),
+    count: vi.fn(),
+  };
+  driver.created = created;
+  return driver as any;
+}
+
+const DOC_SCHEMA = {
+  name: 'doc',
+  fields: {
+    title: { type: 'text' },
+    doc_no: { type: 'autonumber', required: true, format: 'D-{0000}' },
+  },
+};
+
+describe('ObjectQL autonumber ownership (#1603)', () => {
+  let engine: ObjectQL;
+
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.mocked(SchemaRegistry.getObject).mockReturnValue(DOC_SCHEMA as any);
+    engine = new ObjectQL();
+  });
+
+  it('defers to a native-autonumber driver (does NOT pre-fill) and passes required-validation', async () => {
+    const driver = makeDriver(true);
+    engine.registerDriver(driver, true);
+    await engine.init();
+
+    const result = await engine.insert('doc', { title: 'Spec' });
+
+    // Engine did NOT generate the value — the row handed to the driver has no doc_no.
+    expect(driver.created).toHaveLength(1);
+    expect(driver.created[0].doc_no).toBeUndefined();
+    // Engine never scanned for a max via the fallback seed path.
+    expect(driver.find).not.toHaveBeenCalled();
+    // The driver's value comes back to the caller.
+    expect(result.doc_no).toBe('D-0001');
+  });
+
+  it('falls back to engine generation for a driver without native autonumber', async () => {
+    const driver = makeDriver(false);
+    engine.registerDriver(driver, true);
+    await engine.init();
+
+    const result = await engine.insert('doc', { title: 'Spec' });
+
+    // Engine pre-filled the value before calling the driver.
+    expect(driver.created).toHaveLength(1);
+    expect(driver.created[0].doc_no).toBe('D-0001');
+    expect(result.doc_no).toBe('D-0001');
+  });
+});

packages/objectql/src/engine.ts

@@ -719,21 +719,27 @@ export class ObjectQL implements IDataEngine {
   }
 
   /**
-   * Generate values for empty `autonumber` fields on insert. Runs BEFORE
-   * required-validation so a `required` autonumber (e.g. a record number) is
-   * never rejected for "missing" — the runtime owns the value, not the client.
+   * Generate values for empty `autonumber` fields on insert — ONLY for drivers
+   * that do not generate them natively (memory, mongodb). For SQL-backed objects
+   * the driver owns a persistent, atomic `_objectstack_sequences` table and
+   * advertises `supports.autonumber === true`; the engine then defers entirely
+   * and never pre-fills (so the persistent sequence is the single source of
+   * truth — see #1603). Required-validation exempts `autonumber` either way, so
+   * a `required` record number is never rejected for "missing" — the runtime
+   * owns the value, not the client.
    *
-   * The next value is `max(existing) + 1`, seeded once per `object.field` from
-   * the store then incremented in memory (monotonic within the process,
-   * resilient to deletions). `autonumberFormat` is honored, e.g.
-   * `CASE-{0000}` → `CASE-0042`. NOTE: in-memory seeding is single-instance;
-   * a persistent sequence store is a follow-up for multi-instance setups.
+   * In the fallback path the next value is `max(existing) + 1`, seeded once per
+   * `object.field` from the store then incremented in memory (monotonic within
+   * the process, resilient to deletions). `autonumberFormat` is honored, e.g.
+   * `CASE-{0000}` → `CASE-0042`. NOTE: this in-memory seeding is single-instance.
    */
   private async applyAutonumbers(
     object: string,
     record: Record<string, unknown>,
     execCtx?: ExecutionContext,
+    driverOwnsAutonumber?: boolean,
   ): Promise<void> {
+    if (driverOwnsAutonumber) return; // driver generates persistently in create()
     const fields = (this.getSchema(object) as any)?.fields;
     if (!fields || typeof fields !== 'object' || Array.isArray(fields)) return;
     for (const [name, def] of Object.entries(fields)) {
@@ -745,7 +751,10 @@ export class ObjectQL implements IDataEngine {
       if (next == null) next = await this.seedAutonumber(object, name, execCtx);
       next += 1;
       this.autonumberCounters.set(key, next);
-      record[name] = this.formatAutonumber((def as any).autonumberFormat, next);
+      // Honor either the spec-canonical `autonumberFormat` or the shorthand
+      // `format` (both appear in metadata; the driver reads both too) — #1603.
+      const fmt = (def as any).autonumberFormat ?? (def as any).format;
+      record[name] = this.formatAutonumber(fmt, next);
     }
   }
 
@@ -1834,13 +1843,16 @@ export class ObjectQL implements IDataEngine {
         let result;
         const nowSnap = new Date();
         const schemaForValidation = this._registry.getObject(object);
+        // When the driver generates autonumbers natively (persistent SQL
+        // sequence), the engine defers to it — see #1603.
+        const driverOwnsAutonumber = (driver as any)?.supports?.autonumber === true;
         if (Array.isArray(hookContext.input.data)) {
           // Bulk Create — apply defaults per row
           const rows = (hookContext.input.data as any[]).map((row) =>
             this.applyFieldDefaults(object, row as Record<string, unknown>, opCtx.context, nowSnap),
           );
           for (const r of rows) {
-            await this.applyAutonumbers(object, r as Record<string, unknown>, opCtx.context);
+            await this.applyAutonumbers(object, r as Record<string, unknown>, opCtx.context, driverOwnsAutonumber);
           }
           for (const r of rows) {
             await this.encryptSecretFields(object, r, opCtx.context, hookContext.input.options);
@@ -1862,7 +1874,7 @@ export class ObjectQL implements IDataEngine {
             opCtx.context,
             nowSnap,
           );
-          await this.applyAutonumbers(object, row, opCtx.context);
+          await this.applyAutonumbers(object, row, opCtx.context, driverOwnsAutonumber);
           await this.encryptSecretFields(object, row, opCtx.context, hookContext.input.options);
           validateRecord(schemaForValidation, row, 'insert');
           evaluateValidationRules(schemaForValidation as any, row, 'insert', { logger: this.logger });

packages/objectql/src/validation/record-validator.test.ts

@@ -0,0 +1,35 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect } from 'vitest';
+import { validateRecord } from './record-validator.js';
+
+/**
+ * Required-field validation, with the autonumber exemption (#1603).
+ *
+ * `autonumber` values are runtime-owned — the SQL driver assigns them from a
+ * persistent sequence AFTER record-validation runs — so a missing value on an
+ * insert must NOT be reported as a client "required" error.
+ */
+describe('validateRecord — required + autonumber exemption', () => {
+  const schema = {
+    fields: {
+      title: { type: 'text', required: true },
+      record_no: { type: 'autonumber', required: true, format: 'REC-{0000}' },
+    },
+  };
+
+  it('does NOT reject a missing required autonumber on insert', () => {
+    // title supplied, record_no omitted → only the autonumber is missing.
+    expect(() => validateRecord(schema, { title: 'Hello' }, 'insert')).not.toThrow();
+  });
+
+  it('still rejects a missing required NON-autonumber field on insert', () => {
+    expect(() => validateRecord(schema, { record_no: 'REC-0001' }, 'insert')).toThrow(/title/i);
+  });
+
+  it('accepts an explicitly-provided autonumber value', () => {
+    expect(() =>
+      validateRecord(schema, { title: 'Hello', record_no: 'REC-0042' }, 'insert'),
+    ).not.toThrow();
+  });
+});

packages/objectql/src/validation/record-validator.ts

@@ -111,7 +111,10 @@ function optionValues(options: FieldDef['options']): string[] {
 
 function validateOne(name: string, def: FieldDef, value: unknown): FieldValidationError | null {
   // ── required ────────────────────────────────────────────────────
-  if (def.required && isMissing(value)) {
+  // `autonumber` is runtime-owned: the value is generated by the engine /
+  // driver (the SQL driver assigns it from a persistent sequence AFTER this
+  // validation runs), so a missing value is never a client error — see #1603.
+  if (def.required && isMissing(value) && def.type !== 'autonumber') {
     return { field: name, code: 'required', message: `${name} is required` };
   }
   if (isMissing(value)) return null; // nothing else to check

packages/plugins/driver-sql/src/sql-driver-autonumber.test.ts

@@ -170,6 +170,53 @@ describe('SqlDriver auto_number sequence', () => {
     expect(r.case_number).toBe('CASE-001');
   });
 
+  it('fills autonumber fields on bulkCreate (engine defers to the driver — #1603)', async () => {
+    await driver.initObjects([
+      {
+        name: 'contract',
+        fields: {
+          organization_id: { type: 'string' },
+          contract_number: { type: 'autonumber', format: 'CTR-{0000}' },
+          name: { type: 'string' },
+        },
+      },
+    ]);
+
+    const rows = await driver.bulkCreate('contract', [
+      { organization_id: 'org_bulk', name: 'B1' },
+      { organization_id: 'org_bulk', name: 'B2' },
+      { organization_id: 'org_bulk', name: 'B3' },
+    ]);
+
+    const numbers = (rows as any[]).map((r) => r.contract_number).sort();
+    expect(numbers).toEqual(['CTR-0001', 'CTR-0002', 'CTR-0003']);
+
+    // A subsequent single create continues the same persistent sequence.
+    const next = await driver.create('contract', { organization_id: 'org_bulk', name: 'B4' });
+    expect(next.contract_number).toBe('CTR-0004');
+  });
+
+  it('advertises native autonumber support so the engine defers to it', () => {
+    expect(driver.supports.autonumber).toBe(true);
+  });
+
+  it('honors the spec-canonical `autonumberFormat` key as well as `format`', async () => {
+    await driver.initObjects([
+      {
+        name: 'ticket',
+        fields: {
+          // spec-canonical key (field.zod.ts) rather than the `format` shorthand
+          ticket_no: { type: 'autonumber', autonumberFormat: 'TK-{00000}' },
+        },
+      },
+    ]);
+
+    const r1 = await driver.create('ticket', {});
+    const r2 = await driver.create('ticket', {});
+    expect(r1.ticket_no).toBe('TK-00001');
+    expect(r2.ticket_no).toBe('TK-00002');
+  });
+
   it('falls back to options.tenantId when the row has no tenant field value', async () => {
     await driver.initObjects([
       {

packages/plugins/driver-sql/src/sql-driver.ts

@@ -130,6 +130,10 @@ export class SqlDriver implements IDataDriver {
       jsonFields: true,
       arrayFields: true,
       vectorSearch: false,
+      // Persistent, atomic autonumber sequences via `_objectstack_sequences`
+      // (see fillAutoNumberFields / getNextSequenceValue). The engine defers
+      // autonumber generation to this driver — it is the single source of truth.
+      autonumber: true,
 
       // Schema Management
       schemaSync: true,
@@ -702,7 +706,12 @@ export class SqlDriver implements IDataDriver {
   async bulkCreate(object: string, data: any[], options?: DriverOptions): Promise<any> {
     this.auditMissingTenant(object, 'bulkCreate', options);
     for (const row of data) {
-      if (row && typeof row === 'object') this.injectTenantOnInsert(object, row, options);
+      if (row && typeof row === 'object') {
+        this.injectTenantOnInsert(object, row, options);
+        // Reserve a persistent sequence value for each row's autonumber
+        // field(s) — the engine no longer pre-fills these (see #1603).
+        await this.fillAutoNumberFields(object, row, options);
+      }
     }
     const builder = this.getBuilder(object, options);
     return await builder.insert(data).returning('*');
@@ -1068,9 +1077,12 @@ export class SqlDriver implements IDataDriver {
             (this.datetimeFields[tableName] ??= new Set()).add(name);
           }
           if (type === 'auto_number' || type === 'autonumber') {
-            const fmt = typeof field.format === 'string' && field.format
-              ? field.format
-              : '{0000}';
+            // Honor either the spec-canonical `autonumberFormat` or the
+            // shorthand `format` (both appear in metadata) — see #1603.
+            const rawFmt = (typeof field.autonumberFormat === 'string' && field.autonumberFormat)
+              ? field.autonumberFormat
+              : (typeof field.format === 'string' && field.format ? field.format : '');
+            const fmt = rawFmt || '{0000}';
             const m = fmt.match(/\{(0+)\}/);
             const padWidth = m ? m[1].length : 4;
             const prefix = m ? fmt.slice(0, m.index ?? 0) : fmt;

packages/spec/src/data/driver.zod.ts

@@ -227,6 +227,22 @@ export const DriverCapabilitiesSchema = lazySchema(() => z.object({
    */
   vectorSearch: z.boolean().default(false).describe('Supports vector embeddings and similarity search'),
 
+  /**
+   * Whether the driver natively generates persistent autonumber / sequence
+   * values inside `create()` / `bulkCreate()` / `upsert()` (e.g. a DB-backed
+   * `_objectstack_sequences` table that survives restarts and is atomic across
+   * concurrent writers / instances).
+   *
+   * When true, the ObjectQL engine MUST NOT pre-fill `autonumber` fields with
+   * its own in-memory counter — the driver owns generation as the single,
+   * persistent source of truth, and required-validation exempts the field
+   * because the value is assigned after validation. When false/absent (memory,
+   * mongodb), the engine falls back to its in-memory generator.
+   *
+   * Optional so existing driver capability objects need not be updated.
+   */
+  autonumber: z.boolean().optional().describe('Driver natively generates persistent autonumber/sequence values'),
+
   // ============================================================================
   // Schema Management
   // ============================================================================