Skip to content
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,13 @@ export default Sentry.withSentry(
messages: [{ role: 'user', content: 'What is the weather in SF?' }],
});

const stream = await compiled.stream({
messages: [{ role: 'user', content: 'Stream the weather in SF' }],
});
for await (const _chunk of stream) {
// Consuming the iterator is what runs the graph and completes the agent span.
}

return new Response(JSON.stringify({ success: true }));
},
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,10 @@ import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_PIPELINE_NAME,
GEN_AI_RESPONSE_FINISH_REASONS,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TEXT,
GEN_AI_USAGE_INPUT_TOKENS,
GEN_AI_USAGE_OUTPUT_TOKENS,
GEN_AI_USAGE_TOTAL_TOKENS,
Expand All @@ -17,7 +20,7 @@ import { getSpanOp, getSpansFromEnvelope } from '../../../spanUtils';
// want to test that the instrumentation does not break in our
// cloudflare SDK.

it('traces langgraph compile and invoke operations', async ({ signal }) => {
it('traces langgraph invoke and stream operations', async ({ signal }) => {
const runner = createRunner(__dirname)
.ignore('event')
.expect(envelope => {
Expand All @@ -26,10 +29,13 @@ it('traces langgraph compile and invoke operations', async ({ signal }) => {
expect(segmentSpan?.name).toBe('GET /');

const genAiSpans = spans.filter(span => getSpanOp(span)?.startsWith('gen_ai.'));
expect(genAiSpans).toHaveLength(1);
expect(genAiSpans.map(span => span.name).sort()).toEqual(['invoke_agent weather_assistant']);
expect(genAiSpans).toHaveLength(2);
expect(genAiSpans.map(span => span.name).sort()).toEqual([
'invoke_agent weather_assistant',
'invoke_agent weather_assistant',
]);

const invokeAgentSpan = genAiSpans.find(span => span.name === 'invoke_agent weather_assistant');
const invokeAgentSpan = genAiSpans.find(span => span.attributes[GEN_AI_RESPONSE_STREAMING] === undefined);
expect(invokeAgentSpan).toBeDefined();
expect(invokeAgentSpan!.status).toBe('ok');
expect(invokeAgentSpan!.attributes[GEN_AI_OPERATION_NAME]).toEqual({
Expand Down Expand Up @@ -66,6 +72,38 @@ it('traces langgraph compile and invoke operations', async ({ signal }) => {
type: 'integer',
value: 30,
});

const streamSpan = genAiSpans.find(span => span.attributes[GEN_AI_RESPONSE_STREAMING]?.value === true);
expect(streamSpan).toBeDefined();
expect(streamSpan!.status).toBe('ok');
expect(streamSpan!.attributes[GEN_AI_INPUT_MESSAGES]).toEqual({
type: 'string',
value: '[{"role":"user","content":"Stream the weather in SF"}]',
});
expect(streamSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toEqual({
type: 'string',
value: '[{"role":"assistant","content":"Mock response from LangGraph agent"}]',
});
expect(streamSpan!.attributes[GEN_AI_RESPONSE_MODEL]).toEqual({
type: 'string',
value: 'mock-model',
});
expect(streamSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS]).toEqual({
type: 'array',
value: ['stop'],
});
expect(streamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toEqual({
type: 'integer',
value: 20,
});
expect(streamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS]).toEqual({
type: 'integer',
value: 10,
});
expect(streamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toEqual({
type: 'integer',
value: 30,
});
})
.start(signal);
await runner.makeRequest('get', '/');
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
import { ChatAnthropic } from '@langchain/anthropic';
import { HumanMessage, SystemMessage } from '@langchain/core/messages';
import { createReactAgent } from '@langchain/langgraph/prebuilt';
import * as Sentry from '@sentry/node';
import express from 'express';

function startMockAnthropicServer() {
const app = express();
app.use(express.json());

app.post('/v1/messages', (req, res) => {
res.json({
id: 'msg_react_agent_stream_123',
type: 'message',
role: 'assistant',
content: [{ type: 'text', text: 'Paris is the capital of France.' }],
model: req.body.model,
stop_reason: 'end_turn',
stop_sequence: null,
usage: { input_tokens: 20, output_tokens: 10 },
});
});

return new Promise(resolve => {
const server = app.listen(0, () => resolve(server));
});
}

async function run() {
const server = await startMockAnthropicServer();
const baseUrl = `http://localhost:${server.address().port}`;

await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
const llm = new ChatAnthropic({
model: 'claude-3-5-sonnet-20241022',
apiKey: 'mock-api-key',
clientOptions: { baseURL: baseUrl },
});
const agent = createReactAgent({ llm, tools: [], name: 'helpful_assistant' });

const stream = await agent.stream({
messages: [new SystemMessage('You are a helpful assistant.'), new HumanMessage('What is the capital of France?')],
});
for await (const _chunk of stream) {
// Consume the stream so the agent and its child spans complete before the root span ends.
}
});

await Sentry.flush(2000);
server.close();
}

run();
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,16 @@ async function run() {
{ role: 'user', content: 'Tell me about the weather' },
],
});

const stream = await graph.stream({
messages: [{ role: 'user', content: 'Stream the weather forecast' }],
});
await stream.pipeTo(new WritableStream());

const canceledStream = await graph.stream({
messages: [{ role: 'user', content: 'Cancel the weather forecast' }],
});
await canceledStream.cancel('no longer needed');
});

await Sentry.flush(2000);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import {
GEN_AI_INPUT_MESSAGES,
GEN_AI_OPERATION_NAME,
GEN_AI_PIPELINE_NAME,
GEN_AI_RESPONSE_FINISH_REASONS,
GEN_AI_RESPONSE_MODEL,
GEN_AI_RESPONSE_STREAMING,
GEN_AI_RESPONSE_TEXT,
GEN_AI_RESPONSE_TOOL_CALLS,
GEN_AI_SYSTEM_INSTRUCTIONS,
Expand All @@ -31,14 +33,16 @@ describe('LangGraph integration', () => {
.expect({ transaction: { transaction: 'langgraph-test' } })
.expect({
span: container => {
expect(container.items).toHaveLength(2);
expect(container.items).toHaveLength(4);
expect(container.items.map(span => span.name).sort()).toEqual([
'invoke_agent weather_assistant',
'invoke_agent weather_assistant',
'invoke_agent weather_assistant',
'invoke_agent weather_assistant',
]);

const invokeAgentSpans = container.items.filter(span => span.name === 'invoke_agent weather_assistant');
expect(invokeAgentSpans).toHaveLength(2);
expect(invokeAgentSpans).toHaveLength(4);
for (const span of invokeAgentSpans) {
expect(span.status).toBe('ok');
expect(span.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent');
Expand All @@ -47,6 +51,11 @@ describe('LangGraph integration', () => {
expect(span.attributes[GEN_AI_AGENT_NAME].value).toBe('weather_assistant');
expect(span.attributes[GEN_AI_PIPELINE_NAME].value).toBe('weather_assistant');
}

const streamSpans = invokeAgentSpans.filter(
span => span.attributes[GEN_AI_RESPONSE_STREAMING]?.value === true,
);
expect(streamSpans).toHaveLength(2);
},
})
.start()
Expand All @@ -60,7 +69,7 @@ describe('LangGraph integration', () => {
.expect({ transaction: { transaction: 'langgraph-test' } })
.expect({
span: container => {
expect(container.items).toHaveLength(2);
expect(container.items).toHaveLength(4);

const weatherTodaySpan = container.items.find(span =>
getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES]?.value)?.includes(
Expand All @@ -82,6 +91,30 @@ describe('LangGraph integration', () => {
expect(weatherDetailsSpan!.name).toBe('invoke_agent weather_assistant');
expect(weatherDetailsSpan!.status).toBe('ok');
expect(weatherDetailsSpan!.attributes['sentry.op'].value).toBe('gen_ai.invoke_agent');

const weatherStreamSpan = container.items.find(span =>
getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES]?.value)?.includes(
'Stream the weather forecast',
),
);
expect(weatherStreamSpan).toBeDefined();
expect(weatherStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true);
expect(weatherStreamSpan!.attributes[GEN_AI_RESPONSE_TEXT].value).toBe(
'[{"role":"assistant","content":"Mock LLM response"}]',
);
expect(weatherStreamSpan!.attributes[GEN_AI_RESPONSE_MODEL].value).toBe('mock-model');
expect(weatherStreamSpan!.attributes[GEN_AI_RESPONSE_FINISH_REASONS].value).toEqual(['stop']);
expect(weatherStreamSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS].value).toBe(20);
expect(weatherStreamSpan!.attributes[GEN_AI_USAGE_OUTPUT_TOKENS].value).toBe(10);
expect(weatherStreamSpan!.attributes[GEN_AI_USAGE_TOTAL_TOKENS].value).toBe(30);

const canceledStreamSpan = container.items.find(span =>
getStringAttributeValue(span.attributes[GEN_AI_INPUT_MESSAGES]?.value)?.includes(
'Cancel the weather forecast',
),
);
expect(canceledStreamSpan).toBeDefined();
expect(canceledStreamSpan!.attributes[GEN_AI_RESPONSE_STREAMING].value).toBe(true);
},
})
.start()
Expand Down Expand Up @@ -297,6 +330,36 @@ describe('LangGraph integration', () => {
});
});

createEsmAndCjsTests(__dirname, 'agent-stream-scenario.mjs', 'instrument-agent.mjs', (createRunner, test) => {
test('parents the streaming agent chat span under its agent span', { timeout: 30000 }, async () => {
await createRunner()
.expect({ transaction: { transaction: 'main' } })
.expect({
span: container => {
const agentSpans = container.items.filter(
span => span.attributes[SENTRY_OP]?.value === 'gen_ai.invoke_agent',
);
const chatSpans = container.items.filter(span => span.attributes[SENTRY_OP]?.value === 'gen_ai.chat');
expect(agentSpans).toHaveLength(1);
expect(chatSpans).toHaveLength(1);

const agentSpan = agentSpans[0]!;
const chatSpan = chatSpans[0]!;
expect(agentSpan.name).toBe('invoke_agent helpful_assistant');
expect(agentSpan.status).toBe('ok');
expect(agentSpan.attributes[GEN_AI_RESPONSE_STREAMING]?.value).toBe(true);
expect(agentSpan.span_id).toMatch(/^[\da-f]{16}$/);
expect(chatSpan.parent_span_id).toBe(agentSpan.span_id);
expect(chatSpan.trace_id).toBe(agentSpan.trace_id);
expect(chatSpan.status).toBe('ok');
expect(chatSpan.attributes[GEN_AI_AGENT_NAME]?.value).toBe('helpful_assistant');
},
})
.start()
.completed();
});
});

// createReactAgent with tools - verifies tool execution spans (asserted order-independently, see above).
createEsmAndCjsTests(__dirname, 'agent-tools-scenario.mjs', 'instrument-agent.mjs', (createRunner, test) => {
test('should create tool execution spans for createReactAgent with tools', { timeout: 30000 }, async () => {
Expand Down
Loading
Loading