fix(rest): the import template stars a column only when the import refuses a blank in it

.changeset/18386-template-required-option-default.md

@@ -0,0 +1,12 @@
+---
+'@objectstack/rest': patch
+---
+
+fix(rest): the import template (`GET /api/v1/data/:object/export?template=true`) puts ` *` on a column only when the import refuses a row that leaves it blank
+
+Clause-②: no
+
+- A required field whose option list marks an option `default: true` no longer gets ` *` in the header, and the instructions sheet lists it as not required. A blank cell in that column is imported as the marked option.
+- A required field that declares `defaultValue: null` now gets ` *`, unless one of its options is marked `default: true`: the import treats `null` as no default and refuses the blank. A required field whose default is `''`, or `[]` on a multi-valued field, now gets ` *` too: the required check refuses that default.
+- A required `system`, `readonly` or `autonumber` field named in `?fields=` no longer gets ` *`: the import does not refuse a blank in it.
+- Each dropdown's error title, which is the column header, is cut to 32 characters, the longest title Excel's data-validation dialog takes.

packages/rest/src/import-template-route.test.ts

@@ -18,6 +18,7 @@ import { SqlDriver } from '@objectstack/driver-sql';
 import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol';
 import { RestServer } from './rest-server';
 import { parseXlsxToRows } from './import-prepare.js';
+import { isTemplateRequired, templateInsertDefault } from './import-template.js';
 import { loadXlsxWorkbook } from './xlsx-test-loader.js';
 
 function makeSqliteDriver() {
@@ -341,6 +342,129 @@ describe('?template=true — the template filled in and imported back', () => {
   });
 });
 
+// ---------------------------------------------------------------------------
+// The `*`: starred exactly when the engine refuses a blank
+// ---------------------------------------------------------------------------
+
+/** A required select whose option is marked `default: true` — the showcase task's `status` shape. */
+const TICKET = {
+  name: 'ticket', label: 'Ticket',
+  fields: {
+    title: { name: 'title', type: 'text', label: 'Title', required: true },
+    status: {
+      name: 'status', type: 'select', label: 'Status', required: true,
+      options: [{ label: 'Backlog', value: 'backlog', default: true }, { label: 'Done', value: 'done' }],
+    },
+  },
+};
+
+/** Import rows through the real door and return the report. */
+async function importRows(importRoute: any, object: string, body: Record<string, unknown>) {
+  const out = makeRes();
+  await importRoute.handler({ params: { object }, body } as any, out.res);
+  return out.json();
+}
+
+describe('?template=true — a required field the engine fills from its option default', () => {
+  it('is not starred, its instructions row says not required, and a blank cell imports to the default', async () => {
+    const { get, importRoute, engine } = await boot();
+    engine.registry.registerObject(TICKET as any);
+    await engine.syncSchemas();
+
+    const bytes = (await get({ template: 'true' }, 'ticket')).body();
+    const wb = await loadXlsxWorkbook(bytes);
+    const headers = (wb.worksheets[0].getRow(1).values as unknown[]).slice(1) as string[];
+    expect(headers).toEqual(['Title *', 'Status']);
+    const guide = wb.worksheets[1];
+    expect([guide.getCell('A6').value, guide.getCell('B6').value, guide.getCell('D6').value]).toEqual(['Status', 'status', 'No']);
+    expect([guide.getCell('A5').value, guide.getCell('D5').value]).toEqual(['Title *', 'Yes']);
+
+    // Fill the template the way a user does: a title, the Status cell left blank.
+    const sheet = wb.worksheets[0];
+    sheet.getCell('A2').value = 'Blank status';
+    sheet.getCell('B2').value = null;
+    const filled = Buffer.from(await wb.xlsx.writeBuffer());
+    const report = await importRows(importRoute, 'ticket', {
+      format: 'xlsx', xlsxBase64: filled.toString('base64'), mapping: { 'Title *': 'title', Status: 'status' },
+    });
+    expect(report, JSON.stringify(report)).toMatchObject({ total: 1, ok: 1, errors: 0, created: 1 });
+    const [stored] = await engine.find('ticket', {});
+    expect(stored).toMatchObject({ title: 'Blank status', status: 'backlog' });
+  });
+});
+
+/**
+ * `isTemplateRequired` mirrors the engine's insert-time default
+ * (`ObjectQL.applyFieldDefaults` + `resolveOptionDefault`, both private, so
+ * the template cannot call them). This battery holds the mirror to the engine:
+ * for every shape, the template's own header is starred exactly when the real
+ * import door refuses a blank cell, and a filled blank stores the default the
+ * mirror predicts.
+ */
+const OPTS = [{ label: 'A', value: 'a' }, { label: 'B', value: 'b' }, { label: 'C', value: 'c' }];
+const mark = (...values: string[]) => OPTS.map((o) => (values.includes(o.value) ? { ...o, default: true } : o));
+
+const PARITY: Record<string, { def: Record<string, unknown>; blank: 'refused' | 'accepted'; stored?: unknown }> = {
+  plain: { def: { type: 'text' }, blank: 'refused' },
+  default_value: { def: { type: 'text', defaultValue: 'x' }, blank: 'accepted', stored: 'x' },
+  default_value_false: { def: { type: 'boolean', defaultValue: false }, blank: 'accepted', stored: false },
+  option_default: { def: { type: 'select', options: mark('b') }, blank: 'accepted', stored: 'b' },
+  option_default_first_wins: { def: { type: 'select', options: mark('a', 'c') }, blank: 'accepted', stored: 'a' },
+  option_default_multiselect: { def: { type: 'multiselect', options: mark('a', 'c') }, blank: 'accepted', stored: ['a', 'c'] },
+  option_default_select_multiple: { def: { type: 'select', multiple: true, options: mark('b') }, blank: 'accepted', stored: ['b'] },
+  option_default_any_type: { def: { type: 'text', options: [{ label: 'X', value: 'x', default: true }] }, blank: 'accepted', stored: 'x' },
+  no_marked_option: { def: { type: 'select', options: OPTS }, blank: 'refused' },
+  marked_option_without_value: { def: { type: 'select', options: [{ label: 'A', default: true }, { label: 'B', value: 'b' }] }, blank: 'refused' },
+  non_canonical_flag: { def: { type: 'select', options: OPTS.map((o) => ({ ...o, default: 'true' })) }, blank: 'refused' },
+  default_value_null: { def: { type: 'text', defaultValue: null }, blank: 'refused' },
+  default_value_null_option_default: { def: { type: 'select', defaultValue: null, options: mark('c') }, blank: 'accepted', stored: 'c' },
+  default_value_beats_option: { def: { type: 'select', defaultValue: 'a', options: mark('b') }, blank: 'accepted', stored: 'a' },
+  default_value_empty_string: { def: { type: 'text', defaultValue: '' }, blank: 'refused' },
+  default_value_empty_array: { def: { type: 'multiselect', defaultValue: [], options: mark('a') }, blank: 'refused' },
+  readonly: { def: { type: 'text', readonly: true }, blank: 'accepted' },
+  system: { def: { type: 'text', system: true }, blank: 'accepted' },
+  autonumber: { def: { type: 'autonumber', autonumberFormat: 'N-{0000}' }, blank: 'accepted' },
+};
+
+describe('the `*` agrees with the engine: starred exactly when the import door refuses a blank', () => {
+  it('for every shape of default the engine reads', async () => {
+    const { get, importRoute, engine } = await boot();
+    const objectOf = (key: string) => `pty_${key}`;
+    for (const [key, { def }] of Object.entries(PARITY)) {
+      engine.registry.registerObject({
+        name: objectOf(key), label: key,
+        fields: { title: { name: 'title', type: 'text' }, f: { name: 'f', label: 'F', required: true, ...def } },
+      } as any);
+    }
+    await engine.syncSchemas();
+
+    const disagreements: string[] = [];
+    for (const [key, { def, blank, stored }] of Object.entries(PARITY)) {
+      const object = objectOf(key);
+      // The template's own verdict — its header, via `?fields=` so every shape is a column.
+      const wb = await loadXlsxWorkbook((await get({ template: 'true', fields: 'f' }, object)).body());
+      const starred = wb.worksheets[0].getCell('A1').value === 'F *';
+      expect(wb.worksheets[1].getCell('D5').value, key).toBe(starred ? 'Yes' : 'No');
+      expect(isTemplateRequired({ required: true, ...def }), key).toBe(starred);
+
+      // The engine's verdict — a blank cell through the real import door.
+      const report = await importRows(importRoute, object, { format: 'json', rows: [{ title: key, f: '' }] });
+      const refused = report?.errors === 1;
+      if (refused) expect(report.results?.[0], key).toMatchObject({ ok: false, field: 'f', code: 'required' });
+      else expect(report, `${key}: ${JSON.stringify(report)}`).toMatchObject({ ok: 1, errors: 0, created: 1 });
+
+      if (starred !== refused) disagreements.push(`${key}: template ${starred ? 'starred' : 'unstarred'}, engine ${refused ? 'refused' : 'accepted'} the blank`);
+      expect(refused ? 'refused' : 'accepted', key).toBe(blank);
+      if (!refused && stored !== undefined) {
+        const [row] = await engine.find(object, {});
+        expect(row?.f, key).toEqual(stored);
+        expect(templateInsertDefault({ required: true, ...def }), key).toEqual(stored);
+      }
+    }
+    expect(disagreements).toEqual([]);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Without ?template=true: byte for byte the pre-change output
 // ---------------------------------------------------------------------------

packages/rest/src/import-template.test.ts

@@ -19,13 +19,16 @@ import {
   TEMPLATE_INAPPLICABLE_PARAMS,
   TEMPLATE_READER_CLAIMS,
   TEMPLATE_VALIDATED_ROWS,
+  TEMPLATE_VALIDATION_ERROR_MAX,
+  TEMPLATE_VALIDATION_ERROR_TITLE_MAX,
   buildImportTemplateWorkbook,
   columnLetter,
   describeTemplateColumns,
   isTemplateRequired,
   readTemplateMode,
   resolveTemplateProjection,
   templateColumns,
+  templateInsertDefault,
   templateText,
 } from './import-template.js';
 import {
@@ -168,18 +171,61 @@ describe('resolveTemplateProjection — the write projection, and the stated fal
 });
 
 describe('isTemplateRequired — the `*` mark', () => {
+  const STAGES = [{ label: 'Backlog', value: 'backlog', default: true }, { label: 'Done', value: 'done' }];
+
   it('marks a required field with no default', () => {
     expect(isTemplateRequired({ required: true })).toBe(true);
   });
   it('does NOT mark a required field that declares a default — the engine fills it', () => {
     expect(isTemplateRequired({ required: true, defaultValue: 'standard' })).toBe(false);
   });
+  it('does NOT mark a required field whose option is marked `default: true` — the engine fills that too', () => {
+    expect(isTemplateRequired({ required: true, type: 'select', options: STAGES })).toBe(false);
+    expect(isTemplateRequired({ required: true, type: 'multiselect', options: STAGES })).toBe(false);
+  });
+  it('marks one whose option list fills nothing: no marked option, a non-canonical flag, or a marked option with no value', () => {
+    expect(isTemplateRequired({ required: true, type: 'select', options: [{ label: 'A', value: 'a' }] })).toBe(true);
+    expect(isTemplateRequired({ required: true, type: 'select', options: [{ label: 'A', value: 'a', default: 'true' }] })).toBe(true);
+    expect(isTemplateRequired({ required: true, type: 'select', options: [{ label: 'A', default: true }] })).toBe(true);
+  });
+  it('`defaultValue: null` declares no default (the engine\'s `== null`), so the option list decides', () => {
+    expect(isTemplateRequired({ required: true, defaultValue: null })).toBe(true);
+    expect(isTemplateRequired({ required: true, type: 'select', defaultValue: null, options: STAGES })).toBe(false);
+  });
+  it('marks one whose default the required check counts as empty — `\'\'`, or `[]` on a multi-valued field', () => {
+    expect(isTemplateRequired({ required: true, type: 'text', defaultValue: '' })).toBe(true);
+    expect(isTemplateRequired({ required: true, type: 'multiselect', defaultValue: [], options: STAGES })).toBe(true);
+  });
+  it('does not mark a field the insert never runs `required` on: system, readonly, autonumber', () => {
+    expect(isTemplateRequired({ required: true, system: true })).toBe(false);
+    expect(isTemplateRequired({ required: true, readonly: true })).toBe(false);
+    expect(isTemplateRequired({ required: true, type: 'autonumber' })).toBe(false);
+  });
   it('does not mark an optional field', () => {
     expect(isTemplateRequired({ required: false })).toBe(false);
     expect(isTemplateRequired({})).toBe(false);
   });
 });
 
+describe('templateInsertDefault — the default the engine fills on insert', () => {
+  it('`defaultValue` wins over a marked option; `\'\'` is a real default, `null` is none', () => {
+    const options = [{ value: 'a' }, { value: 'b', default: true }];
+    expect(templateInsertDefault({ defaultValue: 'x', options })).toBe('x');
+    expect(templateInsertDefault({ defaultValue: '', options })).toBe('');
+    expect(templateInsertDefault({ defaultValue: null, options })).toBe('b');
+  });
+  it('a single-valued field takes the FIRST marked option; a multi-valued one takes every marked option', () => {
+    const options = [{ value: 'a', default: true }, { value: 'b' }, { value: 'c', default: true }];
+    expect(templateInsertDefault({ type: 'select', options })).toBe('a');
+    expect(templateInsertDefault({ type: 'select', multiple: true, options })).toEqual(['a', 'c']);
+    expect(templateInsertDefault({ type: 'multiselect', options })).toEqual(['a', 'c']);
+  });
+  it('no default at all answers undefined', () => {
+    expect(templateInsertDefault(undefined)).toBeUndefined();
+    expect(templateInsertDefault({ type: 'select', options: ['a', 'b'] })).toBeUndefined();
+  });
+});
+
 // ─────────────────────────────────────────────────────────────────────────────
 // Reading the request
 // ─────────────────────────────────────────────────────────────────────────────
@@ -374,6 +420,26 @@ describe('describeTemplateColumns', () => {
     }
   });
 
+  it('a required select whose option is marked `default: true` is not starred, and its instructions row says not required', async () => {
+    const schema = {
+      fields: {
+        title: { name: 'title', type: 'text', label: 'Title', required: true },
+        status: {
+          name: 'status', type: 'select', label: 'Status', required: true,
+          options: [{ label: 'Backlog', value: 'backlog', default: true }, { label: 'Done', value: 'done' }],
+        },
+      },
+    };
+    for (const [locale, no] of [['en', 'No'], ['zh-CN', '否']] as const) {
+      const cols = describeTemplateColumns(schema, ['title', 'status'], { locale });
+      expect(cols.map((c) => [c.header, c.required])).toEqual([['Title *', true], ['Status', false]]);
+      const wb = await loadXlsxWorkbook(Buffer.from(await (await buildImportTemplateWorkbook(cols, { locale })).xlsx.writeBuffer()));
+      expect(wb.worksheets[0].getCell('B1').value, locale).toBe('Status');
+      expect(wb.worksheets[1].getCell('A6').value, locale).toBe('Status');
+      expect(wb.worksheets[1].getCell('D6').value, locale).toBe(no);
+    }
+  });
+
   it('a name in ?fields= that is no field of the object is described as such', () => {
     const [unknown] = describeTemplateColumns(KITCHEN_SINK, ['nope'], { locale: 'en' });
     expect(unknown).toMatchObject({ header: 'nope', type: '', howToFill: 'Not a field of this object.' });
@@ -436,6 +502,30 @@ describe('buildImportTemplateWorkbook — read back', () => {
     expect(sheet.getCell('E2').dataValidation).toBeUndefined();
   });
 
+  it('a dropdown\'s error title is cut to Excel\'s 32 characters, and its message to 255', async () => {
+    const longLabel = 'Preferred contact channel for renewal outreach'; // 46 characters
+    const schema = {
+      fields: {
+        channel: {
+          name: 'channel', type: 'select', label: longLabel, required: true,
+          options: [{ label: 'Email', value: 'email' }, { label: 'Phone', value: 'phone' }],
+        },
+        short: { name: 'short', type: 'boolean', label: 'Hot' },
+      },
+    };
+    const cols = describeTemplateColumns(schema, ['channel', 'short'], { locale: 'en' });
+    expect(cols[0].header.length).toBeGreaterThan(TEMPLATE_VALIDATION_ERROR_TITLE_MAX);
+    const wb = await buildImportTemplateWorkbook(cols, { locale: 'en' });
+    const back = await loadXlsxWorkbook(Buffer.from(await wb.xlsx.writeBuffer()));
+    const long = back.worksheets[0].getCell('A2').dataValidation;
+    expect(TEMPLATE_VALIDATION_ERROR_TITLE_MAX).toBe(32);
+    expect(long?.errorTitle?.length).toBeLessThanOrEqual(32);
+    expect(long?.errorTitle).toBe(`${longLabel} *`.slice(0, 32));
+    expect(long?.error?.length).toBeLessThanOrEqual(TEMPLATE_VALIDATION_ERROR_MAX);
+    // A title that fits is written whole.
+    expect(back.worksheets[0].getCell('B2').dataValidation?.errorTitle).toBe('Hot');
+  });
+
   it('the instructions sheet has one row per column: header, field, type, required, how to fill', async () => {
     const { cols, wb } = await roundTrip('en');
     const guide = wb.worksheets[1];