feat(article): falsification the schema judges, not a heuristic

packages/article/package.json

@@ -12,8 +12,10 @@
     "test": "vitest run --coverage"
   },
   "dependencies": {
+    "ai": "^7.0.77",
     "cheerio": "^1.2.0",
-    "domhandler": "^6.0.1"
+    "domhandler": "^6.0.1",
+    "zod": "^4.4.3"
   },
   "devDependencies": {
     "@types/node": "^22.20.1",

packages/article/src/falsify.test.ts

@@ -0,0 +1,240 @@
+// 3.4's criteria: a malformed model answer is rejected by the schema, and a
+// paragraph longer than 1000 characters reaches the model whole.
+//
+// The model is mocked, so these are deterministic. What is being tested is not
+// the model's judgement — that cannot be unit tested — but that the code around
+// it believes only what the schema accepts, and hands over the text it was given.
+import { MockLanguageModelV4 } from 'ai/test';
+import { describe, expect, it } from 'vitest';
+
+import {
+  falsifiableCandidates,
+  falsify,
+  FALSIFICATIONS_PER_ARTICLE,
+  MIN_FALSIFIABLE_CHARS,
+} from './falsify.js';
+
+/** Answers with whatever text it is given, and records the prompt it received. */
+function modelAnswering(text: string) {
+  const prompts: string[] = [];
+  const model = new MockLanguageModelV4({
+    doGenerate: async (options) => {
+      prompts.push(JSON.stringify(options.prompt));
+      return {
+        content: [{ type: 'text' as const, text }],
+        finishReason: { unified: 'stop' as const, raw: undefined },
+        usage: {
+          inputTokens: {
+            total: 1200,
+            noCache: 1200,
+            cacheRead: undefined,
+            cacheWrite: undefined,
+          },
+          outputTokens: { total: 340, text: 340, reasoning: undefined },
+        },
+        warnings: [],
+      };
+    },
+  });
+  return { model, prompts };
+}
+
+function answer(items: readonly Record<string, unknown>[]): string {
+  return JSON.stringify({ falsifications: items });
+}
+
+const CANDIDATE = {
+  index: 2,
+  text: 'Un paragraphe assez long pour être falsifié, largement au-dessus du plancher de cent caractères imposé par le contrat.',
+};
+
+const GOOD = {
+  paragraphIndex: 2,
+  swappedText: 'Un paragraphe modifié.',
+  explanation: 'La vérité.',
+  hint: 'Vérifiez la date.',
+};
+
+describe('which paragraphs are offered', () => {
+  it('keeps the collected index even after dropping the short ones', () => {
+    const paragraphs = ['court', 'x'.repeat(MIN_FALSIFIABLE_CHARS), 'y'.repeat(200)];
+    expect(falsifiableCandidates(paragraphs).map((candidate) => candidate.index)).toEqual(
+      [1, 2],
+    );
+  });
+
+  it('drops a paragraph under the floor, keeps one exactly on it', () => {
+    expect(falsifiableCandidates(['a'.repeat(MIN_FALSIFIABLE_CHARS - 1)])).toEqual([]);
+    expect(falsifiableCandidates(['a'.repeat(MIN_FALSIFIABLE_CHARS)])).toHaveLength(1);
+  });
+
+  it('measures the trimmed text, as the current filter does', () => {
+    expect(
+      falsifiableCandidates([`   ${'a'.repeat(MIN_FALSIFIABLE_CHARS - 1)}   `]),
+    ).toEqual([]);
+  });
+
+  it('keeps the constants the current code holds twice', () => {
+    // D8: `MIN_FALSIFIABLE_CHARS` in settings and `MIN_PARAGRAPH_LENGTH = 100`
+    // hard-coded in `misinformation.py`. One constant here.
+    expect(MIN_FALSIFIABLE_CHARS).toBe(100);
+    expect(FALSIFICATIONS_PER_ARTICLE).toBe(4);
+  });
+});
+
+describe('3.4 — a paragraph is never truncated on the way to the model', () => {
+  // The bug: the current code sends `text[:1000]`, so the model rewrites an
+  // ending it never saw while the player is served the paragraph in full — a
+  // paragraph whose second half contradicts its first.
+  it('sends a 2000-character paragraph whole', async () => {
+    const long = `Début. ${'mot '.repeat(480)}Fin distinctive du paragraphe.`;
+    expect(long.length).toBeGreaterThan(1_000);
+
+    const { model, prompts } = modelAnswering(
+      answer([
+        { ...GOOD, paragraphIndex: 0, swappedText: long.replace('Début', 'Debut faux') },
+      ]),
+    );
+
+    const result = await falsify({
+      model,
+      topic: 'Paris',
+      candidates: [{ index: 0, text: long }],
+    });
+
+    expect(result.ok).toBe(true);
+    // The whole paragraph is in the prompt, tail included.
+    expect(prompts[0]).toContain('Fin distinctive du paragraphe.');
+    expect(JSON.parse(prompts[0] as string)).toBeDefined();
+  });
+
+  it('comes back whole, with no 1000-character cliff', async () => {
+    const long = 'a'.repeat(1_500);
+    const falsified = `${'b'.repeat(1_499)}!`;
+    const { model } = modelAnswering(
+      answer([{ ...GOOD, paragraphIndex: 0, swappedText: falsified }]),
+    );
+
+    const result = await falsify({
+      model,
+      topic: 'Paris',
+      candidates: [{ index: 0, text: long }],
+    });
+
+    expect(result.ok && result.value.falsifications[0]?.swappedText).toHaveLength(1_500);
+  });
+});
+
+describe('3.4 — the schema is the only judge', () => {
+  it('accepts a well-formed answer', async () => {
+    const { model } = modelAnswering(answer([GOOD]));
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+
+    expect(result.ok && result.value.falsifications).toEqual([GOOD]);
+  });
+
+  // Each of these used to be handled by a heuristic. Now they are one path.
+  it.each([
+    ['prose around the JSON', 'Voici le tableau : {"falsifications": []} — voilà.'],
+    ['a Markdown fence', '```json\n{"falsifications":[]}\n```'],
+    ['a bare array', '[{"paragraphIndex":2}]'],
+    ['not JSON at all', 'Je ne peux pas faire cela.'],
+    ['an empty answer', ''],
+    ['an envelope with the wrong key', '{"results":[]}'],
+  ])('rejects %s', async (_name, text) => {
+    const { model } = modelAnswering(text);
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+    expect(result.ok).toBe(false);
+  });
+
+  it.each([
+    ['a missing hint', { paragraphIndex: 2, swappedText: 'x', explanation: 'y' }],
+    ['an empty swappedText', { ...GOOD, swappedText: '' }],
+    ['a string index', { ...GOOD, paragraphIndex: '2' }],
+    ['a fractional index', { ...GOOD, paragraphIndex: 2.5 }],
+    ['a negative index', { ...GOOD, paragraphIndex: -1 }],
+  ])('rejects %s', async (_name, item) => {
+    const { model } = modelAnswering(answer([item]));
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+    expect(result.ok).toBe(false);
+  });
+});
+
+describe('3.4 — an index the model was not given', () => {
+  // The current code falls back to position when the indices do not match, which
+  // turns "the model renumbered things" into "the player is graded on a
+  // paragraph nobody touched". Dropped instead.
+  it('drops a falsification quoting an index that was never offered', async () => {
+    const { model } = modelAnswering(answer([GOOD, { ...GOOD, paragraphIndex: 99 }]));
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+
+    expect(
+      result.ok && result.value.falsifications.map((item) => item.paragraphIndex),
+    ).toEqual([2]);
+  });
+
+  it('fails when nothing it returned was offered', async () => {
+    const { model } = modelAnswering(answer([{ ...GOOD, paragraphIndex: 99 }]));
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+
+    expect(result).toMatchObject({ ok: false, reason: 'unexpected_response' });
+  });
+
+  // C3.3 forbids two falsifications on one paragraph, and the database refuses
+  // to store them. Collapsing here is what keeps that from becoming an error.
+  it('keeps one falsification per paragraph', async () => {
+    const { model } = modelAnswering(
+      answer([GOOD, { ...GOOD, swappedText: 'Une autre version.' }]),
+    );
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+
+    expect(result.ok && result.value.falsifications).toHaveLength(1);
+  });
+
+  it('returns them sorted by paragraph index', async () => {
+    const candidates = [CANDIDATE, { index: 0, text: 'z'.repeat(150) }];
+    const { model } = modelAnswering(
+      answer([GOOD, { ...GOOD, paragraphIndex: 0, swappedText: 'Premier.' }]),
+    );
+    const result = await falsify({ model, topic: 'Paris', candidates });
+
+    expect(
+      result.ok && result.value.falsifications.map((item) => item.paragraphIndex),
+    ).toEqual([0, 2]);
+  });
+});
+
+describe('3.4 — the prompt', () => {
+  it('carries the topic and the original text, verbatim', async () => {
+    const { model, prompts } = modelAnswering(answer([GOOD]));
+    await falsify({ model, topic: 'Chat domestique', candidates: [CANDIDATE] });
+
+    const sent = prompts[0] as string;
+    expect(sent).toContain('Chat domestique');
+    expect(sent).toContain('plancher de cent caract');
+    // The wording is the one in use today, not a rewrite: this phase's pitfall
+    // is not to mix a stack change with a behaviour change.
+    expect(sent).toContain('expert en création de désinformation crédible');
+    expect(sent).toContain('paragraph_index');
+  });
+
+  it('refuses to ask when no paragraph is long enough', async () => {
+    const { model, prompts } = modelAnswering(answer([GOOD]));
+    const result = await falsify({ model, topic: 'Paris', candidates: [] });
+
+    expect(result).toMatchObject({ ok: false, reason: 'unexpected_response' });
+    expect(prompts).toEqual([]);
+  });
+});
+
+describe('3.4 — what the call cost', () => {
+  it('reports the tokens the provider gave', async () => {
+    const { model } = modelAnswering(answer([GOOD]));
+    const result = await falsify({ model, topic: 'Paris', candidates: [CANDIDATE] });
+
+    expect(result.ok && result.value.usage).toEqual({
+      inputTokens: 1200,
+      outputTokens: 340,
+    });
+  });
+});