cache-control.test.js

import { describe, it, beforeEach } from 'node:test';
import assert from 'node:assert/strict';
import { extractCachePolicy } from '../src/handlers/messages.js';
import { handleMessages } from '../src/handlers/messages.js';
import { checkin as poolCheckin, checkout as poolCheckout, poolClear } from '../src/conversation-pool.js';

// Anthropic prompt-caching markers (cache_control: { type: 'ephemeral',
// ttl?: '5m' | '1h' }) appear on tools[], system[] blocks, and
// messages[].content[] blocks. Cascade upstream doesn't speak this
// dialect — the proxy parses, summarises, and strips them so they
// don't leak into Cascade requests, then attributes the resulting
// cache_creation tokens to ephemeral_5m or ephemeral_1h based on the
// presence of any 1h marker.

describe('extractCachePolicy — strip + summarise cache_control markers', () => {
  it('counts 5m markers across tools, system, messages and strips them', () => {
    const body = {
      tools: [
        { name: 't1', cache_control: { type: 'ephemeral' } },
        { name: 't2' },
      ],
      system: [
        { type: 'text', text: 'sys1' },
        { type: 'text', text: 'sys2', cache_control: { type: 'ephemeral', ttl: '5m' } },
      ],
      messages: [
        { role: 'user', content: [
          { type: 'text', text: 'hello' },
          { type: 'text', text: 'tagged', cache_control: { type: 'ephemeral' } },
        ] },
      ],
    };
    const policy = extractCachePolicy(body);
    assert.equal(policy.breakpointCount, 3);
    assert.equal(policy.has1h, false);
    // markers stripped in place
    assert.equal(body.tools[0].cache_control, undefined);
    assert.equal(body.system[1].cache_control, undefined);
    assert.equal(body.messages[0].content[1].cache_control, undefined);
  });

  it('flags has1h when any marker requests 1h ttl', () => {
    const body = {
      system: [
        { type: 'text', text: 'a', cache_control: { type: 'ephemeral', ttl: '5m' } },
        { type: 'text', text: 'b', cache_control: { type: 'ephemeral', ttl: '1h' } },
      ],
    };
    const p = extractCachePolicy(body);
    assert.equal(p.breakpointCount, 2);
    assert.equal(p.has1h, true);
  });

  it('returns zero policy and no mutation when no markers present', () => {
    const body = {
      tools: [{ name: 't' }],
      system: [{ type: 'text', text: 'x' }],
      messages: [{ role: 'user', content: 'hi' }],
    };
    const p = extractCachePolicy(body);
    assert.equal(p.breakpointCount, 0);
    assert.equal(p.has1h, false);
  });

  // C5: a top-level cache_control is NOT part of the official Anthropic schema
  // (breakpoints live on tools/system/content blocks). It is now ignored rather
  // than treated as a whole-request breakpoint, so it contributes no policy and
  // cannot leak downstream (anthropicToOpenAI never spreads the raw body).
  it('ignores an unofficial top-level cache_control field', () => {
    const body = {
      cache_control: { type: 'ephemeral', ttl: '1h' },
      messages: [{ role: 'user', content: 'hi' }],
    };
    const p = extractCachePolicy(body);
    assert.equal(p.breakpointCount, 0);
    assert.equal(p.has1h, false);
  });

  it('does not throw on malformed bodies', () => {
    assert.doesNotThrow(() => extractCachePolicy({}));
    assert.doesNotThrow(() => extractCachePolicy({ tools: null, system: 'x' }));
    assert.doesNotThrow(() => extractCachePolicy({ messages: [{ role: 'user', content: null }] }));
  });
});

describe('handleMessages — cache_control round-trip into Anthropic usage shape', () => {
  function fakeChat(usagePatch) {
    return {
      async handleChatCompletions(body, ctx) {
        // body.__cachePolicy must reach chat.js
        return {
          status: 200,
          body: {
            id: 'chat_1', object: 'chat.completion', created: 1, model: body.model,
            choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
            usage: {
              prompt_tokens: 10, completion_tokens: 1, total_tokens: 11,
              prompt_tokens_details: { cached_tokens: 0 },
              cache_creation_input_tokens: 100,
              cache_read_input_tokens: 0,
              ...usagePatch,
            },
          },
        };
      },
    };
  }

  it('5m markers route creation tokens to ephemeral_5m_input_tokens', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      messages: [
        { role: 'user', content: [
          { type: 'text', text: 'hi', cache_control: { type: 'ephemeral' } },
        ] },
      ],
    }, fakeChat({
      cache_creation_input_tokens: 100,
      cache_creation: { ephemeral_5m_input_tokens: 100, ephemeral_1h_input_tokens: 0 },
    }));
    assert.equal(result.status, 200);
    assert.equal(result.body.usage.cache_creation_input_tokens, 100);
    assert.deepEqual(result.body.usage.cache_creation, {
      ephemeral_5m_input_tokens: 100,
      ephemeral_1h_input_tokens: 0,
    });
  });

  it('1h markers route creation tokens to ephemeral_1h_input_tokens', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      messages: [
        { role: 'user', content: [
          { type: 'text', text: 'hi', cache_control: { type: 'ephemeral', ttl: '1h' } },
        ] },
      ],
    }, fakeChat({
      cache_creation_input_tokens: 200,
      cache_creation: { ephemeral_5m_input_tokens: 0, ephemeral_1h_input_tokens: 200 },
    }));
    assert.equal(result.status, 200);
    assert.equal(result.body.usage.cache_creation_input_tokens, 200);
    assert.deepEqual(result.body.usage.cache_creation, {
      ephemeral_5m_input_tokens: 0,
      ephemeral_1h_input_tokens: 200,
    });
  });

  it('cascade pool entry honours ttlHintMs longer than default', async () => {
    poolClear();
    const baseEntry = {
      cascadeId: 'c1', sessionId: 's1', lsPort: 12345, apiKey: 'k',
      createdAt: Date.now(),
    };
    // Default-TTL entry: should expire at the 30-min default.
    poolCheckin('fp_default', { ...baseEntry });
    // 1h-hint entry: should outlive the default.
    poolCheckin('fp_1h', { ...baseEntry }, '', 90 * 60 * 1000);
    // After 35 min the default entry is gone, the 1h entry remains.
    // We can't fast-forward time without mocking; instead simulate by
    // mutating lastAccess on the stored entries directly via checkout +
    // re-checkin with an old timestamp, but the simpler check is just
    // that the entry struct keeps the hint. Verify by checkout while
    // both are still fresh (< pool default), then by the surface fact
    // that the 1h-hint entry still has its hint after restore.
    const entry = poolCheckout('fp_1h');
    assert.equal(entry?.ttlHintMs, 90 * 60 * 1000);
    poolClear();
  });

  it('cascade pool checkin preserves ttlHintMs when restoring without an explicit hint', () => {
    poolClear();
    const e = { cascadeId: 'c', sessionId: 's', lsPort: 1, apiKey: 'k', ttlHintMs: 90 * 60 * 1000 };
    poolCheckin('fp1', e);
    const got = poolCheckout('fp1');
    assert.equal(got.ttlHintMs, 90 * 60 * 1000);
    poolClear();
  });

  it('emits both flat fields and nested split when no markers were sent', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      messages: [{ role: 'user', content: 'hi' }],
    }, fakeChat({
      cache_creation_input_tokens: 50,
    }));
    assert.equal(result.status, 200);
    const u = result.body.usage;
    // Both shapes coexist; the flat total equals the split sum.
    assert.equal(u.cache_creation_input_tokens, 50);
    assert.equal(u.cache_read_input_tokens, 0);
    assert.equal(
      u.cache_creation.ephemeral_5m_input_tokens + u.cache_creation.ephemeral_1h_input_tokens,
      u.cache_creation_input_tokens,
    );
  });

  // B8: upstream supplies a FLAT cache_creation number with no per-TTL split.
  // When the request marked a 1h prefix, the flat number must route to the 1h
  // bucket (per cachePolicy.has1h) instead of the old unconditional 5m default.
  it('routes an upstream FLAT cache_creation to the 1h bucket when the request marked 1h (B8)', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: 'cached prefix', cache_control: { type: 'ephemeral', ttl: '1h' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, fakeChat({
      // Real upstream number (wins over the estimate), FLAT with no split.
      cache_creation_input_tokens: 321,
      cache_creation: undefined,
    }));
    assert.equal(result.status, 200);
    const u = result.body.usage;
    assert.equal(u.cache_creation_input_tokens, 321);
    assert.equal(u.cache_creation.ephemeral_1h_input_tokens, 321, 'flat number routed to 1h per has1h');
    assert.equal(u.cache_creation.ephemeral_5m_input_tokens, 0);
  });

  it('routes an upstream FLAT cache_creation to the 5m bucket by default (B8)', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: 'cached prefix', cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, fakeChat({
      cache_creation_input_tokens: 210,
      cache_creation: undefined,
    }));
    assert.equal(result.status, 200);
    const u = result.body.usage;
    assert.equal(u.cache_creation_input_tokens, 210);
    assert.equal(u.cache_creation.ephemeral_5m_input_tokens, 210, 'flat number defaults to 5m when no 1h marker');
    assert.equal(u.cache_creation.ephemeral_1h_input_tokens, 0);
  });
});

// When the upstream reports NO cache tokens (DEVIN_CONNECT free tier never
// does), but the request marked a cacheable prefix with cache_control, the
// proxy falls back to a LOCAL, CJK-aware estimate of the prefix size so
// Claude Code sees a non-zero, deterministic cache_creation_input_tokens
// instead of a misleading 0. A real upstream number always wins over the
// estimate. (unverified: local estimate — true values need a paid account to
// calibrate; see PAID ledger task E.)
describe('extractCachePolicy — local cache-prefix token estimate', () => {
  it('estimates the cumulative prefix at the deepest breakpoint', () => {
    const body = {
      system: [
        { type: 'text', text: 'aaaa aaaa aaaa aaaa' },
        { type: 'text', text: 'bbbb bbbb', cache_control: { type: 'ephemeral' } },
      ],
      messages: [
        { role: 'user', content: [{ type: 'text', text: 'not cached, after the breakpoint' }] },
      ],
    };
    const p = extractCachePolicy(body);
    assert.equal(p.breakpointCount, 1);
    // Prefix = both system blocks (cumulative up to + including the marked one),
    // NOT the trailing user message after the breakpoint.
    assert.ok(p.estCacheCreationTokens > 0);
    const prefixOnly =
      Math.ceil('aaaa aaaa aaaa aaaa'.length / 4) + Math.ceil('bbbb bbbb'.length / 4);
    assert.equal(p.estCacheCreationTokens, prefixOnly);
  });

  it('weights CJK content ~1 token/char in the prefix estimate', () => {
    const cjk = '你好世界你好世界'; // 8 CJK chars → ~8 tokens
    const ascii = 'abcd'.repeat(2); // 8 ASCII chars → ~2 tokens
    const cjkBody = {
      system: [{ type: 'text', text: cjk, cache_control: { type: 'ephemeral' } }],
    };
    const asciiBody = {
      system: [{ type: 'text', text: ascii, cache_control: { type: 'ephemeral' } }],
    };
    const pc = extractCachePolicy(cjkBody);
    const pa = extractCachePolicy(asciiBody);
    assert.equal(pc.estCacheCreationTokens, 8);
    assert.equal(pa.estCacheCreationTokens, 2);
    // Same char count, but CJK estimated far higher (no ~4× undercount).
    assert.ok(pc.estCacheCreationTokens > pa.estCacheCreationTokens);
  });

  it('includes tool name/description/schema tokens in the prefix', () => {
    const body = {
      tools: [
        {
          name: 'search',
          description: 'find things',
          input_schema: { type: 'object', properties: { q: { type: 'string' } } },
          cache_control: { type: 'ephemeral' },
        },
      ],
    };
    const p = extractCachePolicy(body);
    assert.equal(p.breakpointCount, 1);
    assert.ok(p.estCacheCreationTokens > 0);
  });

  it('returns zero estimate when no breakpoint is present', () => {
    const p = extractCachePolicy({
      system: [{ type: 'text', text: 'plenty of uncached text here' }],
      messages: [{ role: 'user', content: 'hi' }],
    });
    assert.equal(p.breakpointCount, 0);
    assert.equal(p.estCacheCreationTokens, 0);
  });
});

describe('handleMessages — local cache estimate fallback when upstream reports none', () => {
  // A cacheable prefix that clears the C2 minimum (Sonnet/Opus ~1024 tokens).
  // ASCII estimates at ~chars/4, so ~5000 chars → ~1250 tokens > 1024.
  const BIG_ASCII_PREFIX = 'cached system prompt block here. '.repeat(160); // >1024 est
  // Upstream returns NO usage at all (DEVIN_CONNECT free tier shape) — no
  // prompt_tokens, no cache fields — so the local estimate is the only source
  // and the B7 clamp (min against promptTotal) is a no-op here.
  function bareUsageChat() {
    return {
      async handleChatCompletions(body) {
        return {
          status: 200,
          body: {
            id: 'chat_1', object: 'chat.completion', created: 1, model: body.model,
            choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
            usage: { completion_tokens: 1 },
          },
        };
      },
    };
  }

  it('passes __cachePolicy through to chat.js when a breakpoint is present', async () => {
    let seenPolicy;
    await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: 'cached system prompt block here', cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, {
      async handleChatCompletions(body) {
        seenPolicy = body.__cachePolicy;
        return {
          status: 200,
          body: {
            id: 'c', object: 'chat.completion', created: 1, model: body.model,
            choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
            usage: { prompt_tokens: 10, completion_tokens: 1, total_tokens: 11 },
          },
        };
      },
    });
    assert.ok(seenPolicy, '__cachePolicy must reach chat.js');
    assert.ok(seenPolicy.estCacheCreationTokens > 0);
  });

  it('fills cache_creation_input_tokens from the local estimate (5m default)', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: BIG_ASCII_PREFIX, cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    assert.equal(result.status, 200);
    const u = result.body.usage;
    // Non-zero estimate instead of a misleading 0 (prefix clears the C2 floor).
    assert.ok(u.cache_creation_input_tokens > 0);
    assert.equal(u.cache_read_input_tokens, 0);
    // Default ttl 5m bucket carries the estimate.
    assert.equal(u.cache_creation.ephemeral_5m_input_tokens, u.cache_creation_input_tokens);
    assert.equal(u.cache_creation.ephemeral_1h_input_tokens, 0);
    // Invariant preserved.
    assert.equal(
      u.cache_creation.ephemeral_5m_input_tokens + u.cache_creation.ephemeral_1h_input_tokens,
      u.cache_creation_input_tokens,
    );
  });

  // C2: a small cacheable prefix (below the model minimum) writes NO cache
  // entry — the emitted cache_creation must be 0, not a phantom estimate. The
  // breakpoint is still counted for policy (pool TTL hint) upstream.
  it('floors sub-threshold prefixes to 0 cache_creation (C2)', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: 'tiny cached prefix', cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    assert.equal(result.status, 200);
    const u = result.body.usage;
    assert.equal(u.cache_creation_input_tokens, 0, 'prefix under the ~1024 min → 0 creation');
    assert.equal(u.cache_creation.ephemeral_5m_input_tokens, 0);
    assert.equal(u.cache_creation.ephemeral_1h_input_tokens, 0);
  });

  it('routes the estimate to the 1h bucket when a 1h marker is present', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: BIG_ASCII_PREFIX, cache_control: { type: 'ephemeral', ttl: '1h' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    assert.equal(result.status, 200);
    const u = result.body.usage;
    assert.ok(u.cache_creation_input_tokens > 0);
    assert.equal(u.cache_creation.ephemeral_1h_input_tokens, u.cache_creation_input_tokens);
    assert.equal(u.cache_creation.ephemeral_5m_input_tokens, 0);
    assert.equal(
      u.cache_creation.ephemeral_5m_input_tokens + u.cache_creation.ephemeral_1h_input_tokens,
      u.cache_creation_input_tokens,
    );
  });

  // C6: a request that mixes a 5m and a 1h breakpoint must split the estimated
  // prefix per TTL (each breakpoint owns the incremental prefix since the last
  // one), not collapse everything into the 1h bucket via the old has1h binary.
  it('splits a mixed 5m + 1h prefix estimate per TTL bucket (C6)', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [
        { type: 'text', text: BIG_ASCII_PREFIX, cache_control: { type: 'ephemeral', ttl: '5m' } },
        { type: 'text', text: BIG_ASCII_PREFIX, cache_control: { type: 'ephemeral', ttl: '1h' } },
      ],
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    assert.equal(result.status, 200);
    const u = result.body.usage;
    // Both buckets are populated (mixed TTL), not just the 1h one.
    assert.ok(u.cache_creation.ephemeral_5m_input_tokens > 0, '5m segment attributed');
    assert.ok(u.cache_creation.ephemeral_1h_input_tokens > 0, '1h segment attributed');
    // Mutual exclusion / invariant: buckets sum to the flat total.
    assert.equal(
      u.cache_creation.ephemeral_5m_input_tokens + u.cache_creation.ephemeral_1h_input_tokens,
      u.cache_creation_input_tokens,
    );
    // Two equal segments → roughly balanced split (within rounding).
    assert.ok(
      Math.abs(u.cache_creation.ephemeral_5m_input_tokens - u.cache_creation.ephemeral_1h_input_tokens) <= 2,
      'equal-sized 5m/1h segments split roughly evenly',
    );
  });

  // B6 + B7: when the estimate is substituted, the three buckets
  // (input_tokens / cache_creation / cache_read) must be mutually exclusive —
  // the prefix must not be double-counted into input_tokens — and the estimate
  // must be clamped so cache_creation never exceeds the total prompt.
  it('keeps input/creation/read mutually exclusive and clamps the estimate (B6/B7)', async () => {
    // Upstream reports a prompt_tokens SMALLER than the raw prefix estimate to
    // exercise the clamp; no cache fields so the local estimate is used.
    const clampChat = {
      async handleChatCompletions(body) {
        return {
          status: 200,
          body: {
            id: 'chat_1', object: 'chat.completion', created: 1, model: body.model,
            choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
            usage: { prompt_tokens: 500, completion_tokens: 1, total_tokens: 501 },
          },
        };
      },
    };
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: BIG_ASCII_PREFIX, cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, clampChat);
    assert.equal(result.status, 200);
    const u = result.body.usage;
    // B7: clamped to promptTotal (500), not the larger raw estimate.
    assert.ok(u.cache_creation_input_tokens <= 500, 'creation clamped to promptTotal');
    assert.ok(u.cache_creation_input_tokens > 0);
    // B6: input + creation + read never exceeds the total prompt (no double
    // count of the prefix into input_tokens).
    assert.ok(
      u.input_tokens + u.cache_creation_input_tokens + u.cache_read_input_tokens <= 500,
      'three buckets stay within promptTotal (mutually exclusive)',
    );
  });

  it('CJK cache_control content is estimated with CJK weighting in usage', async () => {
    // CJK weights ~1 token/char, so ~1200 CJK chars clears the C2 floor;
    // the same char count in ASCII (~chars/4) does NOT, so it emits 0. This
    // still demonstrates the CJK weighting (CJK creation > ASCII creation).
    const sameCharCount = 1200;
    const asciiResult = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: 'a'.repeat(sameCharCount), cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    const cjkResult = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: '你'.repeat(sameCharCount), cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    // Same char count but CJK estimated far higher than ASCII.
    assert.ok(
      cjkResult.body.usage.cache_creation_input_tokens >
      asciiResult.body.usage.cache_creation_input_tokens,
    );
  });

  it('does NOT override a real upstream cache number with the estimate', async () => {
    const upstreamChat = {
      async handleChatCompletions(body) {
        return {
          status: 200,
          body: {
            id: 'chat_1', object: 'chat.completion', created: 1, model: body.model,
            choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }],
            usage: {
              prompt_tokens: 10, completion_tokens: 1, total_tokens: 11,
              cache_creation_input_tokens: 999,
              cache_read_input_tokens: 0,
            },
          },
        };
      },
    };
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      system: [{ type: 'text', text: 'cached system prompt block here', cache_control: { type: 'ephemeral' } }],
      messages: [{ role: 'user', content: 'hi' }],
    }, upstreamChat);
    // Real upstream value wins; the estimate is not substituted.
    assert.equal(result.body.usage.cache_creation_input_tokens, 999);
  });

  it('emits no cache fields beyond the existing shape when there is no cache_control (byte-identical behavior)', async () => {
    const result = await handleMessages({
      model: 'claude-sonnet-4.6',
      max_tokens: 16,
      messages: [{ role: 'user', content: 'hi' }],
    }, bareUsageChat());
    const u = result.body.usage;
    // No breakpoint → no estimate → cache fields stay zero exactly as before.
    assert.equal(u.cache_creation_input_tokens, 0);
    assert.equal(u.cache_read_input_tokens, 0);
    assert.equal(u.cache_creation.ephemeral_5m_input_tokens, 0);
    assert.equal(u.cache_creation.ephemeral_1h_input_tokens, 0);
  });
});