fix(objectql): rows-path sum / avg add with Kahan-Babuska-Neumaier compensation, as SQLite does

The engine's rows path (in-memory-aggregation.ts) folded sum and avg
naively, while SQLite 3.43+ compensates, so one query answered two doubles
on SQLite depending on the path engine.aggregate took (0.1 + 0.2 + 0.3:
native 0.6, rows 0.6000000000000001). Both arms now add through one
compensated fold transcribed from SQLite's kahanBabuskaNeumaierStep and
its finalizers' overflow guard.

Claude-Session: https://claude.ai/code/session_01N8TPEsoJxPsdSdNKGnNGEN
Co-authored-by: Claude <noreply@anthropic.com>

Author: claude

packages/objectql/src/in-memory-aggregation-compensated-sum.test.ts

@@ -0,0 +1,148 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#20489] The rows path's `sum` / `avg` add with Kahan-Babuska-Neumaier
+ * compensation (`in-memory-aggregation.ts`, `compensatedSum`), the summation
+ * SQLite 3.43+ uses for its own `sum` / `avg`.
+ *
+ * The expected values below are SQLite's own answers, measured with
+ * better-sqlite3 (SQLite 3.53.4), sql.js (3.49.1) and @libsql/client (3.45.1)
+ * over a `REAL` and a `NUMERIC` column holding the same values. All three
+ * engines agreed on every fixture. The naive fold the rows path used before is
+ * shown beside each one:
+ *
+ * | fixture | naive `sum` / `avg` (before) | SQLite native = compensated (after) |
+ * |:--|:--|:--|
+ * | `0.1, 0.2, 0.3` | `0.6000000000000001` / `0.20000000000000004` | `0.6` / `0.19999999999999998` |
+ * | `1e16, 1, -1e16` | `0` / `0` | `1` / `0.3333333333333333` |
+ * | `1e16, 0.5, -1e16` | `0` / `0` | `0.5` / `0.16666666666666666` |
+ * | `0.1, 0.2` (two addends) | `0.30000000000000004` / `0.15000000000000002` | the same |
+ * | `1, 2, 3, 40, 500` (integers) | `546` / `109.2` | the same |
+ *
+ * `engine.aggregate` and REST on SQLite are pinned in `packages/rest`
+ * (`rest-aggregate-compensated-sum.test.ts`): the rows path and the native
+ * path answer the same double there.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { applyInMemoryAggregation } from './in-memory-aggregation.js';
+
+/** The fold the rows path used before #20489: in order, one addition at a time. */
+const naiveSum = (xs: readonly number[]) => xs.reduce((a, b) => a + b, 0);
+
+/** `sum` and `avg` of `w` over `values`, one group, through the rows path. */
+function sumAvg(values: readonly unknown[]): { s: unknown; a: unknown } {
+  const rows = values.map((w, i) => ({ id: `r${i}`, w }));
+  const [out] = applyInMemoryAggregation(rows, {
+    aggregations: [
+      { function: 'sum', field: 'w', alias: 's' },
+      { function: 'avg', field: 'w', alias: 'a' },
+    ],
+  });
+  return out as { s: unknown; a: unknown };
+}
+
+describe('[#20489] rows path — sum / avg add with compensation, as SQLite does', () => {
+  it("the card's fixture: 0.1 + 0.2 + 0.3 answers SQLite's 0.6, and avg its 0.19999999999999998", () => {
+    const values = [0.1, 0.2, 0.3];
+    // The fixture discriminates: the naive fold answers another double.
+    expect(naiveSum(values)).toBe(0.6000000000000001);
+    expect(sumAvg(values)).toStrictEqual({ s: 0.6, a: 0.19999999999999998 });
+    // So `having { s: { $eq: 0.6 } }` now keeps the group on this path too.
+    expect(sumAvg(values).s === 0.6).toBe(true);
+  });
+
+  it('a mixed-sign set with large cancellation keeps the small addend', () => {
+    expect(naiveSum([1e16, 1, -1e16])).toBe(0);
+    expect(sumAvg([1e16, 1, -1e16])).toStrictEqual({ s: 1, a: 1 / 3 });
+    expect(naiveSum([1e16, 0.5, -1e16])).toBe(0);
+    expect(sumAvg([1e16, 0.5, -1e16])).toStrictEqual({ s: 0.5, a: 0.5 / 3 });
+  });
+
+  it('two addends are unchanged: the compensated a + b is the naive one', () => {
+    for (const pair of [[0.1, 0.2], [0.7, 0.1], [1e16, 1], [-0.3, 0.1]]) {
+      const n = naiveSum(pair);
+      expect(sumAvg(pair), `${pair}`).toStrictEqual({ s: n, a: n / 2 });
+    }
+    expect(sumAvg([0.1, 0.2])).toStrictEqual({ s: 0.30000000000000004, a: 0.15000000000000002 });
+  });
+
+  it('integers whose partial sums stay within 2^53 are unchanged, and stay integers', () => {
+    const values = [1, 2, 3, 40, 500];
+    const { s, a } = sumAvg(values);
+    expect(s).toBe(naiveSum(values));
+    expect(s).toBe(546);
+    expect(Number.isInteger(s)).toBe(true);
+    expect(a).toBe(109.2);
+    expect(sumAvg([-7, 3, 12, 0, 9_000_000_000])).toStrictEqual({ s: 9_000_000_008, a: 9_000_000_008 / 5 });
+  });
+
+  it('above 2^53 the compensated total is the exact one SQLite answers, where the naive fold lost the 1s', () => {
+    // SQLite 3.53.4 answers 9007199254740994 over a REAL and over a NUMERIC column.
+    expect(naiveSum([2 ** 53, 1, 1])).toBe(2 ** 53);
+    expect(sumAvg([2 ** 53, 1, 1])).toStrictEqual({ s: 9007199254740994, a: 3002399751580331.5 });
+  });
+
+  it('under groupBy and a per-aggregation filter, every bucket takes the same fold', () => {
+    const rows = [
+      { g: 'x', k: 'in', w: 0.1 },
+      { g: 'x', k: 'in', w: 0.2 },
+      { g: 'x', k: 'out', w: 100 },
+      { g: 'x', k: 'in', w: 0.3 },
+      { g: 'y', k: 'in', w: 1e16 },
+      { g: 'y', k: 'in', w: 1 },
+      { g: 'y', k: 'in', w: -1e16 },
+    ];
+    const out = applyInMemoryAggregation(rows, {
+      groupBy: ['g'],
+      aggregations: [
+        { function: 'sum', field: 'w', alias: 's', filter: { k: 'in' } },
+        { function: 'avg', field: 'w', alias: 'a', filter: { k: 'in' } },
+        { function: 'sum', field: 'w', alias: 'all' },
+      ],
+    } as any).sort((p, q) => String(p.g).localeCompare(String(q.g)));
+    expect(out).toStrictEqual([
+      { g: 'x', s: 0.6, a: 0.19999999999999998, all: 100.6 },
+      { g: 'y', s: 1, a: 1 / 3, all: 1 },
+    ]);
+  });
+});
+
+describe('[#20489] rows path — what the compensated fold leaves as it was', () => {
+  it('null: sum adds nothing for it, avg leaves it out of the count', () => {
+    expect(sumAvg([0.1, null, 0.2, undefined, 0.3])).toStrictEqual({ s: 0.6, a: 0.19999999999999998 });
+    expect(sumAvg([10, null, 20])).toStrictEqual({ s: 30, a: 15 });
+  });
+
+  it('a non-numeric cell reads as 0 in both, and counts in avg; a numeric string reads as its number', () => {
+    expect(sumAvg([10, 'abc', 20])).toStrictEqual({ s: 30, a: 10 });
+    expect(sumAvg(['0.1', '0.2', '0.3'])).toStrictEqual({ s: 0.6, a: 0.19999999999999998 });
+    expect(sumAvg([true, false, true])).toStrictEqual({ s: 2, a: 2 / 3 });
+  });
+
+  it('an empty group: sum 0, avg null — with no rows, and with only nulls', () => {
+    expect(sumAvg([])).toStrictEqual({ s: 0, a: null });
+    expect(sumAvg([null, undefined])).toStrictEqual({ s: 0, a: null });
+  });
+
+  it('a non-finite total is the naive one, as SQLite returns its running sum when the error term overflows', () => {
+    for (const values of [
+      [Infinity, 1, 2],
+      [1, -Infinity, 0.3],
+      [1e308, 1e308, -1e308],
+      [Infinity, -Infinity, 1],
+      [NaN, 0.1, 0.2],
+    ]) {
+      const n = naiveSum(values);
+      const { s, a } = sumAvg(values);
+      expect(Object.is(s, n), `sum of ${values}: ${s} vs naive ${n}`).toBe(true);
+      expect(Object.is(a, n / values.length), `avg of ${values}`).toBe(true);
+    }
+  });
+
+  it('the answer is a JS number on every arm, never a string', () => {
+    const { s, a } = sumAvg(['0.1', 0.2, '0.3']);
+    expect(typeof s).toBe('number');
+    expect(typeof a).toBe('number');
+  });
+});

packages/objectql/src/in-memory-aggregation.ts

@@ -228,12 +228,15 @@ function aggregateBucket(
       case 'count_distinct':
         out[alias] = new Set(values.filter((v) => v != null)).size;
         break;
+      // [#20489] Both arms add through ONE compensated fold
+      // ({@link compensatedSum}) — the summation SQLite's own `sum` / `avg`
+      // use — so the rows path and SQLite's native path answer the same double.
       case 'sum':
-        out[alias] = values.reduce((a, b) => a + toNumber(b), 0);
+        out[alias] = compensatedSum(values.map(toNumber));
         break;
       case 'avg': {
         const nums = values.filter((v) => v != null).map(toNumber);
-        out[alias] = nums.length === 0 ? null : nums.reduce((a, b) => a + b, 0) / nums.length;
+        out[alias] = nums.length === 0 ? null : compensatedSum(nums) / nums.length;
         break;
       }
       // [#11152] `min`/`max` read a BOOLEAN as the number it is worth (0/1) —
@@ -284,6 +287,42 @@ function toNumber(v: any): number {
   return Number.isFinite(n) ? n : 0;
 }
 
+/**
+ * [#20489] The sum of `nums`, added in order with Kahan-Babuska-Neumaier
+ * compensation — the summation SQLite (3.43 and later) uses for its own `sum`
+ * and `avg`, transcribed from its `kahanBabuskaNeumaierStep` and the
+ * finalizers' overflow guard.
+ *
+ * Why: this fold used to add naively (`reduce((a, b) => a + b, 0)`), so one
+ * query answered two doubles on SQLite depending on the path `engine.aggregate`
+ * took. A `number` column holding `0.1`, `0.2` and `0.3` summed to `0.6`
+ * natively and to `0.6000000000000001` here (`avg` `0.19999999999999998`
+ * against `0.20000000000000004`), and `having { s: { $eq: 0.6 } }` kept the
+ * group on the native path alone. Compensated, the two paths agree, and the
+ * answer is the more accurate one (`1e16 + 1 - 1e16` is `1`, not `0`).
+ *
+ * What does not move: two addends (the compensated `a + b` IS the naive one),
+ * integers whose partial sums stay within 2^53 (every addition is exact), and
+ * a non-finite total. `s` below is exactly the naive running sum; once it
+ * overflows or meets a NaN, the error term is non-finite and the naive answer
+ * is returned as it was, which is SQLite's rule too.
+ *
+ * ⚠️ Residual, stated: PostgreSQL and MySQL add their doubles natively without
+ * compensation, so over three or more fractions their native path can still
+ * differ from this one in the last place. An exact `$eq` on a fractional sum
+ * compares doubles; compare with a range.
+ */
+function compensatedSum(nums: readonly number[]): number {
+  let s = 0;
+  let c = 0;
+  for (const r of nums) {
+    const t = s + r;
+    c += Math.abs(s) > Math.abs(r) ? (s - t) + r : (r - t) + s;
+    s = t;
+  }
+  return Number.isFinite(c) ? s + c : s;
+}
+
 /**
  * Bucket a date-like value into an ISO-formatted period label. Weeks start
  * Monday and use ISO week numbering.

packages/rest/src/rest-aggregate-compensated-sum.test.ts

@@ -0,0 +1,183 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#20489] On SQLite, `sum` / `avg` answer ONE double on both of
+ * `engine.aggregate`'s paths — the native `SqlDriver.aggregate` and the rows
+ * path (`objectql`'s `in-memory-aggregation.ts`, which a filtered sibling
+ * aggregation forces) — through `engine.aggregate` and
+ * `POST /api/v1/data/:object/query`, over a real `SqlDriver`.
+ *
+ * SQLite 3.43+ sums with Kahan-Babuska-Neumaier compensation; the rows path
+ * used to add naively. Measured on the base (`b2b6a0643`), SQLite 3.53.4, a
+ * `number` column, same readings through the engine and REST:
+ *
+ * | group | native `sum` / `avg` | rows path `sum` / `avg` (base) |
+ * |:--|:--|:--|
+ * | `0.1, 0.2, 0.3` | `0.6` / `0.19999999999999998` | `0.6000000000000001` / `0.20000000000000004` |
+ * | `1e16, 1, -1e16` | `1` / `0.3333333333333333` | `0` / `0` |
+ * | `1e16, 0.5, -1e16` | `0.5` / `0.16666666666666666` | `0` / `0` |
+ * | `0.1, 0.2` | `0.30000000000000004` / `0.15000000000000002` | the same |
+ * | `1, 2, 3, 40, 500` | `546` / `109.2` | the same |
+ *
+ * `having { s: { $eq: 0.6 } }` kept the first group on the native path and no
+ * group on the rows path. The rows path now adds with the same compensation,
+ * so every row of that table reads as its native column on both paths.
+ *
+ * ## The dialect axis of THIS file
+ *
+ * SQLite only, and deliberately. PostgreSQL and MySQL add their doubles
+ * natively without compensation (`sql-driver.ts`, `AGGREGATE_ACCUMULATION`), so
+ * over three or more fractions their native path can differ from the rows path
+ * in the last place: that is the residual #20489 states, not a defect a pin
+ * here should hold red. An exact `$eq` on a fractional sum compares doubles.
+ */
+
+import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest';
+import type { EngineAggregateOptions, FilterCondition } from '@objectstack/spec/data';
+import { ObjectQL } from '@objectstack/objectql';
+import { SqlDriver } from '@objectstack/driver-sql';
+import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol';
+import { RestServer } from './rest-server';
+
+const OBJECT = 'rest_agg_20489';
+
+const LEDGER = {
+  name: OBJECT,
+  label: 'Ledger 20489',
+  fields: {
+    g: { name: 'g', type: 'text' as const },
+    w: { name: 'w', type: 'number' as const },
+  },
+};
+
+/** group → its values, and SQLite's native `sum` / `avg` over them. */
+const GROUPS: Record<string, { values: number[]; s: number; a: number }> = {
+  card: { values: [0.1, 0.2, 0.3], s: 0.6, a: 0.19999999999999998 },
+  cancel: { values: [1e16, 1, -1e16], s: 1, a: 0.3333333333333333 },
+  cancel_half: { values: [1e16, 0.5, -1e16], s: 0.5, a: 0.16666666666666666 },
+  two: { values: [0.1, 0.2], s: 0.30000000000000004, a: 0.15000000000000002 },
+  ints: { values: [1, 2, 3, 40, 500], s: 546, a: 109.2 },
+};
+
+function createMockServer() {
+  const noop = () => {};
+  return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} };
+}
+
+function makeRes() {
+  const res: any = {
+    write: () => true, end: () => {},
+    header: () => res,
+    status: (code: number) => { res._status = code; return res; },
+    json: (body: any) => { res._json = body; return res; },
+  };
+  return res;
+}
+
+type Path = 'native' | 'rows';
+
+/**
+ * `native`: SqlDriver aggregates and the engine applies `having` to its
+ * answer. `rows`: a filtered aggregation sends the engine to the rows path,
+ * where it aggregates `find()` rows itself and then applies `having`.
+ */
+function grouped(path: Path, having?: Record<string, unknown>): EngineAggregateOptions {
+  const aggregations: NonNullable<EngineAggregateOptions['aggregations']> = [
+    { function: 'sum', field: 'w', alias: 's' },
+    { function: 'avg', field: 'w', alias: 'a' },
+  ];
+  if (path === 'rows') aggregations.push({ function: 'count', alias: 'fb', filter: { g: { $ne: '' } } });
+  return { groupBy: ['g'], aggregations, ...(having ? { having: having as FilterCondition } : {}) };
+}
+
+const byGroup = (rows: any[]) => Object.fromEntries(rows.map((r) => [r.g, { s: r.s, a: r.a }]));
+const groupsOf = (rows: any[]) => rows.map((r) => r.g).sort();
+
+describe('[#20489] sum / avg — one double on both SQLite paths, engine and REST', () => {
+  let engine: ObjectQL;
+  let driver: SqlDriver;
+  let post: (body: Record<string, unknown>) => Promise<any>;
+
+  beforeAll(async () => {
+    driver = new SqlDriver({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true } as never);
+    engine = new ObjectQL();
+    engine.registerDriver(driver, true);
+    await engine.init();
+    engine.registry.registerObject(LEDGER as any);
+    await engine.syncSchemas();
+    let i = 0;
+    for (const [g, { values }] of Object.entries(GROUPS)) {
+      for (const w of values) await engine.insert(OBJECT, { id: `r${i++}`, g, w } as any);
+    }
+
+    const protocol = new ObjectStackProtocolImplementation(engine as any);
+    const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any);
+    (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' });
+    rest.registerRoutes();
+    const route = rest.getRoutes().find((r: any) => r.method === 'POST' && r.path === '/api/v1/data/:object/query');
+    expect(route).toBeDefined();
+    post = async (body) => {
+      const res = makeRes();
+      // What the wire carries: JSON, both ways.
+      await route!.handler({ params: { object: OBJECT }, body: JSON.parse(JSON.stringify(body)) } as any, res);
+      if (res._json !== undefined) res._json = JSON.parse(JSON.stringify(res._json));
+      return res;
+    };
+  });
+
+  afterAll(async () => {
+    try { await engine?.destroy(); } catch { /* noop */ }
+  });
+
+  it('each path is the one it names: native asks driver.aggregate, rows asks driver.find alone', async () => {
+    const spy = vi.spyOn(driver, 'aggregate');
+    try {
+      await engine.aggregate(OBJECT, grouped('native'));
+      expect(spy, 'native').toHaveBeenCalledTimes(1);
+      spy.mockClear();
+      await engine.aggregate(OBJECT, grouped('rows'));
+      expect(spy, 'rows').not.toHaveBeenCalled();
+    } finally {
+      spy.mockRestore();
+    }
+  });
+
+  it("find() reads back the doubles written, so both paths add the same operands", async () => {
+    const rows = (await engine.find(OBJECT, {})) as Array<{ g: string; w: number }>;
+    for (const [g, { values }] of Object.entries(GROUPS)) {
+      expect(rows.filter((r) => r.g === g).map((r) => r.w).sort((x, y) => x - y), g)
+        .toStrictEqual([...values].sort((x, y) => x - y));
+    }
+  });
+
+  it('sum / avg: SQLite native answers, equal on both paths, through the engine and REST', async () => {
+    const expected = Object.fromEntries(Object.entries(GROUPS).map(([g, { s, a }]) => [g, { s, a }]));
+    for (const path of ['native', 'rows'] as const) {
+      expect(byGroup(await engine.aggregate(OBJECT, grouped(path))), `engine, ${path}`).toStrictEqual(expected);
+      const res = await post(grouped(path) as Record<string, unknown>);
+      expect(res._status ?? 200, JSON.stringify(res._json)).toBe(200);
+      expect(byGroup(res._json.records), `REST, ${path}`).toStrictEqual(expected);
+    }
+  });
+
+  it('having $eq on the fractional sum / avg keeps the same group on both paths', async () => {
+    const KEPT: ReadonlyArray<readonly [Record<string, unknown>, string[]]> = [
+      [{ s: { $eq: 0.6 } }, ['card']],
+      [{ a: { $eq: 0.19999999999999998 } }, ['card']],
+      [{ s: { $eq: 1 } }, ['cancel']],
+      [{ s: { $in: [0.5, 546] } }, ['cancel_half', 'ints']],
+      // The residual, as a double: `0.1 + 0.2` is not `0.3` on any path.
+      [{ s: { $eq: 0.3 } }, []],
+      [{ s: { $eq: 0.30000000000000004 } }, ['two']],
+    ];
+    for (const [having, kept] of KEPT) {
+      for (const path of ['native', 'rows'] as const) {
+        expect(groupsOf(await engine.aggregate(OBJECT, grouped(path, having))), `engine, ${path}, ${JSON.stringify(having)}`)
+          .toEqual(kept);
+        const res = await post(grouped(path, having) as Record<string, unknown>);
+        expect(res._status ?? 200, JSON.stringify(res._json)).toBe(200);
+        expect(groupsOf(res._json.records), `REST, ${path}, ${JSON.stringify(having)}`).toEqual(kept);
+      }
+    }
+  });
+});