Skip to content

Commit c7ed7ca

Browse files
JPeer264claude
JPeer264
and
claude
authored
test(cloudflare): Add Workers AI tracing tests for Clef (#25027)
Cloudflare documents Clef (`@cf/cloudflare/clef`) as a drop-in for Jev with the same `state` + `questions` request shape, so it already goes through the `evaluate` path from #24833. Only Jev had tests, so this adds a Clef integration test and a real-Worker E2E test. The integration mock returns the documented Clef response, which has no `{ state, result }` wrapper (the Jev binding response has one). That makes it cover the unwrapped branch of the evaluate response handling, which the Jev test does not reach. The E2E test calls the real model, so it is the one that confirms what the binding actually returns. The docs do not give a `model` value for the response, so it only asserts that it contains `clef`. Fixes #25011 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1 parent 2a085be commit c7ed7ca

7 files changed

Lines changed: 115 additions & 10 deletions

File tree

‎dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/index.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,15 @@ export default Sentry.withSentry(
4242
return new Response(JSON.stringify(result));
4343
}
4444

45+
if (url.pathname === '/evaluate-clef') {
46+
const result = await ai.run('@cf/cloudflare/clef', {
47+
model: 'clef',
48+
state: 'Checkout has been failing for every customer for the last hour.',
49+
questions: { urgent: { type: 'noul', instructions: 'Is this support request urgent?' } },
50+
});
51+
return new Response(JSON.stringify(result));
52+
}
53+
4554
if (url.pathname === '/stream') {
4655
const stream = (await ai.run('@cf/meta/llama-3.1-8b-instruct', {
4756
messages: [{ role: 'user', content: 'What is the capital of France?' }],

‎dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/mocks.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,15 @@ export class MockAi {
3434
};
3535
}
3636

37+
// Unwrapped, as in the documented Clef output schema, so this covers the path without `{ state, result }`.
38+
if (model === '@cf/cloudflare/clef') {
39+
return {
40+
model: 'clef',
41+
answers: { urgent: { type: 'noul', noul: 0.98 } },
42+
usage: { input_tokens: 412, output_tokens: 1 },
43+
};
44+
}
45+
3746
if (inputs?.stream === true) {
3847
return createSseStream([
3948
'{"response":"The capital "}',

‎dev-packages/cloudflare-integration-tests/suites/tracing/workers-ai/test.ts‎

Lines changed: 58 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,9 @@ import {
1717
SENTRY_SEGMENT_NAME,
1818
SENTRY_TRACE_LIFECYCLE,
1919
SENTRY_ENVIRONMENT,
20+
SENTRY_IS_LOCALHOST,
21+
SENTRY_OP,
22+
SENTRY_ORIGIN,
2023
} from '@sentry/conventions/attributes';
2124
import { SDK_VERSION } from '@sentry/core';
2225
import { expect, it } from 'vitest';
@@ -45,8 +48,8 @@ it('traces a basic Workers AI text generation request', async ({ signal }) => {
4548
status: 'ok',
4649
is_segment: false,
4750
attributes: {
48-
'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
49-
'sentry.op': { value: 'gen_ai.chat', type: 'string' },
51+
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
52+
[SENTRY_OP]: { value: 'gen_ai.chat', type: 'string' },
5053
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
5154
[GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' },
5255
[GEN_AI_REQUEST_MODEL]: { value: '@cf/meta/llama-3.1-8b-instruct', type: 'string' },
@@ -64,7 +67,7 @@ it('traces a basic Workers AI text generation request', async ({ signal }) => {
6467
type: 'string',
6568
value: 'The capital of France is Paris.',
6669
},
67-
'sentry.is_localhost': { value: true, type: 'boolean' },
70+
[SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
6871
[SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
6972
[SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
7073
[SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
@@ -99,8 +102,8 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) =
99102
status: 'ok',
100103
is_segment: false,
101104
attributes: {
102-
'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
103-
'sentry.op': { value: 'gen_ai.chat', type: 'string' },
105+
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
106+
[SENTRY_OP]: { value: 'gen_ai.chat', type: 'string' },
104107
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
105108
[GEN_AI_OPERATION_NAME]: { value: 'chat', type: 'string' },
106109
[GEN_AI_REQUEST_MODEL]: { value: '@cf/meta/llama-3.1-8b-instruct', type: 'string' },
@@ -118,7 +121,7 @@ it('traces a streaming Workers AI text generation request', async ({ signal }) =
118121
type: 'string',
119122
value: 'The capital of France is Paris.',
120123
},
121-
'sentry.is_localhost': { value: true, type: 'boolean' },
124+
[SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
122125
[SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
123126
[SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
124127
[SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
@@ -150,8 +153,8 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
150153
status: 'ok',
151154
is_segment: false,
152155
attributes: {
153-
'sentry.origin': { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
154-
'sentry.op': { value: 'gen_ai.evaluate', type: 'string' },
156+
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
157+
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
155158
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
156159
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
157160
[GEN_AI_REQUEST_MODEL]: { value: 'typesafe/jev', type: 'string' },
@@ -164,7 +167,7 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
164167
type: 'string',
165168
value: '[{"type":"evaluation","answers":{"is_urgent":{"type":"noul","noul":0.97}}}]',
166169
},
167-
'sentry.is_localhost': { value: true, type: 'boolean' },
170+
[SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
168171
[SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
169172
[SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
170173
[SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
@@ -180,6 +183,52 @@ it('traces a TypeSafe Jev evaluation like the TypeSafe integration', async ({ si
180183
await runner.completed();
181184
});
182185

186+
it('traces a Clef evaluation like a TypeSafe Jev evaluation', async ({ signal }) => {
187+
const runner = createRunner(__dirname)
188+
.ignore('event')
189+
.expect(envelope => {
190+
const spans = getSpansFromEnvelope(envelope);
191+
const segmentSpan = spans.find(span => span.is_segment);
192+
193+
const genAiSpans = spans.filter(span => getSpanOp(span)?.startsWith('gen_ai.'));
194+
expect(genAiSpans).toHaveLength(1);
195+
196+
expect(genAiSpans[0]).toEqual(
197+
expect.objectContaining({
198+
name: 'evaluate @cf/cloudflare/clef',
199+
status: 'ok',
200+
is_segment: false,
201+
attributes: {
202+
[SENTRY_ORIGIN]: { value: 'auto.ai.cloudflare.workers_ai', type: 'string' },
203+
[SENTRY_OP]: { value: 'gen_ai.evaluate', type: 'string' },
204+
[GEN_AI_PROVIDER_NAME]: { value: 'cloudflare.workers_ai', type: 'string' },
205+
[GEN_AI_OPERATION_NAME]: { value: 'evaluate', type: 'string' },
206+
[GEN_AI_REQUEST_MODEL]: { value: '@cf/cloudflare/clef', type: 'string' },
207+
[GEN_AI_RESPONSE_MODEL]: { value: 'clef', type: 'string' },
208+
[GEN_AI_USAGE_INPUT_TOKENS]: { value: 412, type: 'integer' },
209+
[GEN_AI_USAGE_OUTPUT_TOKENS]: { value: 1, type: 'integer' },
210+
[GEN_AI_USAGE_TOTAL_TOKENS]: { value: 413, type: 'integer' },
211+
// collect only output messages
212+
[GEN_AI_OUTPUT_MESSAGES]: {
213+
type: 'string',
214+
value: '[{"type":"evaluation","answers":{"urgent":{"type":"noul","noul":0.98}}}]',
215+
},
216+
[SENTRY_IS_LOCALHOST]: { value: true, type: 'boolean' },
217+
[SENTRY_TRACE_LIFECYCLE]: { value: 'stream', type: 'string' },
218+
[SENTRY_SEGMENT_NAME]: { value: segmentSpan!.name, type: 'string' },
219+
[SENTRY_SEGMENT_ID]: { value: segmentSpan!.span_id, type: 'string' },
220+
[SENTRY_SDK_NAME]: { value: 'sentry.javascript.cloudflare', type: 'string' },
221+
[SENTRY_SDK_VERSION]: { value: SDK_VERSION, type: 'string' },
222+
[SENTRY_ENVIRONMENT]: { value: 'production', type: 'string' },
223+
},
224+
}),
225+
);
226+
})
227+
.start(signal);
228+
await runner.makeRequest('get', '/evaluate-clef');
229+
await runner.completed();
230+
});
231+
183232
// The Workers AI integration deliberately does not call `captureException` itself.
184233
// When a `run` call fails, the error must bubble up out of the fetch handler and be
185234
// reported by the top-level Cloudflare instrumentation instead — so it shows up in

‎dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/src/index.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,14 @@ export default {
3838
});
3939
return Response.json({ traceId: spanContext?.traceId });
4040
}
41+
case '/test-workers-ai-clef': {
42+
await env.AI.run('@cf/cloudflare/clef', {
43+
model: 'clef',
44+
state: 'Checkout has been failing for every customer for the last hour.',
45+
questions: { urgent: { type: 'noul', instructions: 'Is this support request urgent?' } },
46+
});
47+
return Response.json({ traceId: spanContext?.traceId });
48+
}
4149
case '/test-span':
4250
return Response.json({ spanId: spanContext?.spanId, traceId: spanContext?.traceId });
4351
case '/test-workflow-sleep': {

‎dev-packages/e2e-tests/test-applications/cloudflare-workers-send-to-sentry/tests/send-to-sentry.test.ts‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,3 +114,24 @@ test('Sends a Workers AI gen_ai.evaluate span for a TypeSafe Jev call to Sentry'
114114
'gen_ai.output.messages': expect.stringContaining('"is_urgent"'),
115115
});
116116
});
117+
118+
test('Sends a Workers AI gen_ai.evaluate span for a Clef call to Sentry', async () => {
119+
const { traceId }: { traceId: string } = JSON.parse(await fetchFromWorker(`${workerUrl}/test-workers-ai-clef`, 200));
120+
121+
console.log(`Polling for gen_ai.evaluate span: sentry trace view ${traceTarget(traceId)}`);
122+
123+
let spanId: string | undefined;
124+
await expect
125+
.poll(() => (spanId = findSpanInTrace(traceId, 'gen_ai.evaluate')?.event_id), EVENT_POLLING_OPTIONS)
126+
.toBeDefined();
127+
128+
await expect
129+
.poll(() => fetchSpanAttributes(traceId, spanId!), EVENT_POLLING_OPTIONS)
130+
.toMatchObject({
131+
'gen_ai.operation.name': 'evaluate',
132+
'gen_ai.request.model': '@cf/cloudflare/clef',
133+
'gen_ai.response.model': expect.stringContaining('clef'),
134+
'gen_ai.input.messages': expect.stringContaining('Checkout has been failing'),
135+
'gen_ai.output.messages': expect.stringContaining('"urgent"'),
136+
});
137+
});

‎packages/server-utils/src/ai/workers-ai/utils.ts‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,12 +35,17 @@ export const WORKERS_AI_OPERATION_SPAN_OPS: Record<WorkersAiOperationName, strin
3535
evaluate: GEN_AI_EVALUATE,
3636
};
3737

38+
/**
39+
* Model ID prefixes of the evaluation models (TypeSafe Jev and Cloudflare Clef), traced as `evaluate` spans.
40+
*/
41+
const EVALUATE_MODEL_PREFIXES = ['typesafe/jev', '@cf/cloudflare/clef'];
42+
3843
/**
3944
* Determine the gen_ai operation name from the model and inputs passed to `AI.run`.
4045
* Workers AI exposes a single `run` method, so we infer the operation from the model ID and the input shape.
4146
*/
4247
export function getOperationName(model: unknown, inputs: unknown): WorkersAiOperationName {
43-
if (typeof model === 'string' && model.startsWith('typesafe/jev')) {
48+
if (typeof model === 'string' && EVALUATE_MODEL_PREFIXES.some(prefix => model.startsWith(prefix))) {
4449
return 'evaluate';
4550
}
4651
if (inputs && typeof inputs === 'object') {

‎packages/server-utils/test/ai/lib/utils/workers-ai-utils.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,10 @@ describe('workers-ai utils', () => {
6565
expect(getOperationName('typesafe/jev-1.13', { state: 'Help!', questions: {} })).toBe('evaluate');
6666
});
6767

68+
it('returns "evaluate" for Cloudflare Clef models', () => {
69+
expect(getOperationName('@cf/cloudflare/clef', { state: 'Help!', questions: {} })).toBe('evaluate');
70+
});
71+
6872
it('does not return "evaluate" for other TypeSafe models', () => {
6973
expect(getOperationName('typesafe/other', { prompt: 'Hello' })).toBe('chat');
7074
});

0 commit comments

Comments
 (0)