fix(server-utils): Preserve raw Groq and Together response bodies (#24906)

Groq and Together automatic instrumentation could consume
`.asResponse()` bodies before application code read them. Reuse the
shared API-promise observer to preserve raw chat, streaming, and
embeddings responses while completing their spans.

Stacked on #24902. Part of #24882.

Co-authored-by: GPT-6 <codex@openai.com>

Author: nicohrubec

dev-packages/node-integration-tests/suites/tracing/groq/scenario.mjs

@@ -130,6 +130,31 @@ async function run() {
       model: 'nomic-embed-text-v1_5',
       input: 'Embedding test!',
     });
+
+    const rawChatResponse = await client.chat.completions
+      .create({
+        model: 'llama-3.3-70b-versatile',
+        messages: [{ role: 'user', content: 'Raw response test!' }],
+      })
+      .asResponse();
+    await rawChatResponse.json();
+
+    const rawStreamResponse = await client.chat.completions
+      .create({
+        model: 'llama-3.1-8b-instant',
+        messages: [{ role: 'user', content: 'Raw stream test!' }],
+        stream: true,
+      })
+      .asResponse();
+    await rawStreamResponse.text();
+
+    const rawEmbeddingsResponse = await client.embeddings
+      .create({
+        model: 'nomic-embed-text-v1_5',
+        input: 'Raw embedding test!',
+      })
+      .asResponse();
+    await rawEmbeddingsResponse.json();
   });
 
   await Sentry.flush(2000);

dev-packages/node-integration-tests/suites/tracing/groq/test.ts

@@ -74,6 +74,25 @@ describe('Groq integration', () => {
             expect(embeddingsSpan!.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe(PROVIDER);
             expect(embeddingsSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(8);
             expect(embeddingsSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined();
+
+            const rawSpans = container.items.filter(
+              s =>
+                s.attributes[GEN_AI_PROVIDER_NAME]?.value === PROVIDER &&
+                s.status === 'ok' &&
+                s.attributes[GEN_AI_RESPONSE_ID] === undefined,
+            );
+            expect(rawSpans).toHaveLength(3);
+            expect(rawSpans.map(s => s.name)).toEqual([
+              'chat llama-3.3-70b-versatile',
+              'chat llama-3.1-8b-instant',
+              'embeddings nomic-embed-text-v1_5',
+            ]);
+            for (const span of rawSpans) {
+              expect(span.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]?.value).toBe(ORIGIN);
+              expect(span.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined();
+              expect(span.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined();
+              expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined();
+            }
           },
         })
         .start()
@@ -100,6 +119,26 @@ describe('Groq integration', () => {
             );
             expect(embeddingsSpan).toBeDefined();
             expect(embeddingsSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value).toContain('Embedding test!');
+
+            const rawChatSpan = container.items.find(
+              s =>
+                s.attributes[GEN_AI_REQUEST_MODEL]?.value === 'llama-3.3-70b-versatile' &&
+                s.attributes[GEN_AI_RESPONSE_ID] === undefined,
+            );
+            expect(rawChatSpan).toBeDefined();
+            expect(rawChatSpan!.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe(
+              '[{"role":"user","content":"Raw response test!"}]',
+            );
+            expect(rawChatSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined();
+
+            const rawEmbeddingsSpan = container.items.find(
+              s =>
+                s.attributes[GEN_AI_REQUEST_MODEL]?.value === 'nomic-embed-text-v1_5' &&
+                s.attributes[GEN_AI_RESPONSE_ID] === undefined,
+            );
+            expect(rawEmbeddingsSpan).toBeDefined();
+            expect(rawEmbeddingsSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value).toContain('Raw embedding test!');
+            expect(rawEmbeddingsSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined();
           },
         })
         .start()

dev-packages/node-integration-tests/suites/tracing/together-ai/scenario.mjs

@@ -129,6 +129,31 @@ async function run() {
       model: 'togethercomputer/m2-bert-80M-8k-retrieval',
       input: 'Embedding test!',
     });
+
+    const rawChatResponse = await client.chat.completions
+      .create({
+        model: 'meta-llama/Llama-3.3-70B-Instruct-Turbo',
+        messages: [{ role: 'user', content: 'Raw response test!' }],
+      })
+      .asResponse();
+    await rawChatResponse.json();
+
+    const rawStreamResponse = await client.chat.completions
+      .create({
+        model: 'meta-llama/Llama-3.1-8B-Instruct-Turbo',
+        messages: [{ role: 'user', content: 'Raw stream test!' }],
+        stream: true,
+      })
+      .asResponse();
+    await rawStreamResponse.text();
+
+    const rawEmbeddingsResponse = await client.embeddings
+      .create({
+        model: 'togethercomputer/m2-bert-80M-8k-retrieval',
+        input: 'Raw embedding test!',
+      })
+      .asResponse();
+    await rawEmbeddingsResponse.json();
   });
 
   await Sentry.flush(2000);

dev-packages/node-integration-tests/suites/tracing/together-ai/test.ts

@@ -74,6 +74,25 @@ describe('Together integration', () => {
             expect(embeddingsSpan!.attributes[GEN_AI_PROVIDER_NAME]?.value).toBe(PROVIDER);
             expect(embeddingsSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]?.value).toBe(8);
             expect(embeddingsSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]).toBeUndefined();
+
+            const rawSpans = container.items.filter(
+              s =>
+                s.attributes[GEN_AI_PROVIDER_NAME]?.value === PROVIDER &&
+                s.status === 'ok' &&
+                s.attributes[GEN_AI_RESPONSE_ID] === undefined,
+            );
+            expect(rawSpans).toHaveLength(3);
+            expect(rawSpans.map(s => s.name)).toEqual([
+              'chat meta-llama/Llama-3.3-70B-Instruct-Turbo',
+              'chat meta-llama/Llama-3.1-8B-Instruct-Turbo',
+              'embeddings togethercomputer/m2-bert-80M-8k-retrieval',
+            ]);
+            for (const span of rawSpans) {
+              expect(span.attributes[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]?.value).toBe(ORIGIN);
+              expect(span.attributes[GEN_AI_RESPONSE_MODEL]).toBeUndefined();
+              expect(span.attributes[GEN_AI_USAGE_TOTAL_TOKENS]).toBeUndefined();
+              expect(span.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined();
+            }
           },
         })
         .start()
@@ -100,6 +119,26 @@ describe('Together integration', () => {
             );
             expect(embeddingsSpan).toBeDefined();
             expect(embeddingsSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value).toContain('Embedding test!');
+
+            const rawChatSpan = container.items.find(
+              s =>
+                s.attributes[GEN_AI_REQUEST_MODEL]?.value === 'meta-llama/Llama-3.3-70B-Instruct-Turbo' &&
+                s.attributes[GEN_AI_RESPONSE_ID] === undefined,
+            );
+            expect(rawChatSpan).toBeDefined();
+            expect(rawChatSpan!.attributes[GEN_AI_INPUT_MESSAGES]?.value).toBe(
+              '[{"role":"user","content":"Raw response test!"}]',
+            );
+            expect(rawChatSpan!.attributes[GEN_AI_RESPONSE_TEXT]).toBeUndefined();
+
+            const rawEmbeddingsSpan = container.items.find(
+              s =>
+                s.attributes[GEN_AI_REQUEST_MODEL]?.value === 'togethercomputer/m2-bert-80M-8k-retrieval' &&
+                s.attributes[GEN_AI_RESPONSE_ID] === undefined,
+            );
+            expect(rawEmbeddingsSpan).toBeDefined();
+            expect(rawEmbeddingsSpan!.attributes[GEN_AI_EMBEDDINGS_INPUT]?.value).toContain('Raw embedding test!');
+            expect(rawEmbeddingsSpan!.attributes[GEN_AI_USAGE_INPUT_TOKENS]).toBeUndefined();
           },
         })
         .start()

packages/server-utils/src/integrations/openai-compatible.ts

@@ -9,6 +9,7 @@ import {
 } from '@sentry/core';
 import { GEN_AI_PROVIDER_NAME } from '@sentry/conventions/attributes';
 import { getGenAiSpanOp, resolveAIRecordingOptions } from '../ai/core/utils';
+import { onApiPromiseResponse } from '../ai/core/apiPromise';
 import { addRequestAttributes, extractRequestAttributes } from '../ai/openai';
 import { instrumentStream } from '../ai/openai/streaming';
 import type { OpenAiOptions } from '../ai/openai/types';
@@ -35,8 +36,7 @@ export interface OpenAiCompatibleProvider {
 
 /**
  * The context orchestrion shares across the tracing-channel lifecycle hooks: `arguments` is the live args
- * array passed to `Completions.create(body, options)`, and Node's `tracingChannel` attaches `result` when
- * the returned promise settles.
+ * array passed to `Completions.create(body, options)`, and `result` holds its returned `APIPromise`.
  */
 interface OpenAiCompatibleChannelContext {
   arguments: unknown[];
@@ -67,8 +67,17 @@ export function createOpenAiCompatibleIntegration<T extends OpenAiCompatibleProv
           beforeSpanEnd: (span, data) => {
             addResponseAttributes(span, data.result, resolveAIRecordingOptions(options).recordOutputs);
           },
-          // Streaming: the result is a `Stream` consumed later, so instrument it and let it end the span.
-          deferSpanEnd: ({ span, data }) => wrapStreamResult(span, data, options),
+          deferSpanEnd: ({ span, data, end }) =>
+            onApiPromiseResponse(
+              data.result,
+              response => {
+                data.result = response;
+                if (!wrapStreamResult(span, data, options)) {
+                  end();
+                }
+              },
+              end,
+            ) || wrapStreamResult(span, data, options),
         },
       );
     }
@@ -135,7 +144,7 @@ function isAsyncIterable(value: unknown): value is AsyncIterableStream {
 /**
  * For a streaming `create({ stream: true })` the result is a `Stream` the caller consumes later. We can't
  * swap what `create` returns, but the `Stream` in `data.result` is the same instance the caller holds and
- * `asyncEnd` fires before the caller iterates — so we patch its async iterator in place to run through
+ * we observe it before the caller iterates — so we patch its async iterator in place to run through
  * `instrumentStream`, which accumulates the streamed attributes and ends the span when iteration finishes.
  * Only a streaming call resolves to an async-iterable, so that check alone distinguishes it. Returns `true`
  * to hand span-ending ownership to `instrumentStream`; `false` for non-streaming/errored results, which end

packages/server-utils/src/orchestrion/config/openai-compatible.ts

@@ -4,8 +4,8 @@ import type { InstrumentationConfig } from '../apmTypes';
  * Many providers (Groq, Together, ...) ship a Stainless-generated SDK that mirrors the classic `openai`
  * file layout: `resources/chat/completions.{js,mjs}` (class `Completions`) and
  * `resources/embeddings.{js,mjs}` (class `Embeddings`), each with a `create(body, options)` that returns
- * a thenable `APIPromise` — so `kind: 'Auto'` resolves to `wrapPromise`, and a streaming call resolves to
- * the same OpenAI-style async-iterable `Stream` the openai integration already knows how to consume. These
+ * a thenable `APIPromise`. Sync tracing preserves its lazy body parsing; Auto would trigger it via `.then()`.
+ * A streaming call resolves to the same OpenAI-style async-iterable `Stream` the openai integration consumes. These
  * SDKs ship dual CJS/ESM and the matcher compares `filePath` exactly, hence one entry per built file.
  *
  * Because the wire format is OpenAI-compatible, the span building, streaming and response parsing are all
@@ -16,12 +16,12 @@ export function openAiCompatibleConfig(module: { name: string; versionRange: str
     ...['resources/chat/completions.js', 'resources/chat/completions.mjs'].map(filePath => ({
       channelName: 'chat',
       module: { ...module, filePath },
-      functionQuery: { className: 'Completions', methodName: 'create', kind: 'Auto' as const },
+      functionQuery: { className: 'Completions', methodName: 'create', kind: 'Sync' as const },
     })),
     ...['resources/embeddings.js', 'resources/embeddings.mjs'].map(filePath => ({
       channelName: 'embeddings',
       module: { ...module, filePath },
-      functionQuery: { className: 'Embeddings', methodName: 'create', kind: 'Auto' as const },
+      functionQuery: { className: 'Embeddings', methodName: 'create', kind: 'Sync' as const },
     })),
   ];
 }