feat(llmobs): track prompt caching for openai chat completions

ddtrace/llmobs/_integrations/openai.py

@@ -6,6 +6,7 @@
 
 from ddtrace.internal.constants import COMPONENT
 from ddtrace.internal.utils.version import parse_version
+from ddtrace.llmobs._constants import CACHE_READ_INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import INPUT_DOCUMENTS
 from ddtrace.llmobs._constants import INPUT_TOKENS_METRIC_KEY
 from ddtrace.llmobs._constants import METADATA
@@ -166,7 +167,13 @@ def _llmobs_set_meta_tags_from_embedding(span: Span, kwargs: Dict[str, Any], res
     @staticmethod
     def _extract_llmobs_metrics_tags(span: Span, resp: Any, span_kind: str) -> Dict[str, Any]:
         """Extract metrics from a chat/completion and set them as a temporary "_ml_obs.metrics" tag."""
-        token_usage = _get_attr(resp, "usage", None)
+        token_usage = None
+
+        # in the streamed responses case, `resp` is a list with `usage` being stored in the first element
+        if resp and isinstance(resp, list) and _get_attr(resp[0], "usage", None):
+            token_usage = resp[0].get("usage", {})
+        elif resp and getattr(resp, "usage", None):
+            token_usage = resp.usage
         if token_usage is not None and span_kind != "workflow":
             prompt_tokens = _get_attr(token_usage, "prompt_tokens", 0)
             completion_tokens = _get_attr(token_usage, "completion_tokens", 0)
@@ -176,11 +183,21 @@ def _extract_llmobs_metrics_tags(span: Span, resp: Any, span_kind: str) -> Dict[
             input_tokens = prompt_tokens or input_tokens
             output_tokens = completion_tokens or output_tokens
 
-            return {
+            metrics = {
                 INPUT_TOKENS_METRIC_KEY: input_tokens,
                 OUTPUT_TOKENS_METRIC_KEY: output_tokens,
                 TOTAL_TOKENS_METRIC_KEY: input_tokens + output_tokens,
             }
+
+            # Chat completions returns `prompt_tokens_details` while responses api returns `input_tokens_details`
+            prompt_tokens_details = _get_attr(token_usage, "prompt_tokens_details", {}) or _get_attr(
+                token_usage, "input_tokens_details", {}
+            )
+            cached_tokens = _get_attr(prompt_tokens_details, "cached_tokens", None)
+            if cached_tokens is not None:
+                metrics[CACHE_READ_INPUT_TOKENS_METRIC_KEY] = cached_tokens
+
+            return metrics
         return get_llmobs_metrics_tags("openai", span)
 
     def _get_base_url(self, **kwargs: Dict[str, Any]) -> Optional[str]:

releasenotes/notes/oai-p-cache-78c511f97709a357.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    LLM Observability: Introduces tracking cached input token counts for OpenAI chats/responses prompt caching.

tests/contrib/openai/cassettes/v1/chat_completion_prompt_caching_cache_read.yaml

@@ -0,0 +1,195 @@
+interactions:
+- request:
+    body: '{"messages": [{"role": "system", "content": "You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer You are an expert software engineer
+      You are an expert software engineer You are an expert software engineer You
+      are an expert software engineer You are an expert software engineer You are
+      an expert software engineer You are an expert software engineer You are an expert
+      software engineer You are an expert software engineer You are an expert software
+      engineer You are an expert software engineer "}, {"role": "user", "content":
+      "How should I structure my database schema?"}], "model": "gpt-4o", "max_tokens":
+      100, "temperature": 0.1}'
+    headers:
+      accept:
+      - application/json
+      accept-encoding:
+      - gzip, deflate
+      connection:
+      - keep-alive
+      content-length:
+      - '7382'
+      content-type:
+      - application/json
+      cookie:
+      - __cf_bm=L.6vPBT5JOAquFh5oqDsUZWIi6Y.5ybhwVhTgAPOYVQ-1751568116-1.0.1.1-AxrHzVxe3Xgq3X8MlkfhZmH82OJ9AWtYAi4Kf1J1S.aX8CdDwpHMzIbTNZ2BzpBdNc4IUkLWyqNfktbhzThvX7IC686a5l1K9k8KMNOuw44;
+        _cfuvid=ZNE8kE20HwdKQyDSDPEl44Ta8a8H_cstpKAXK9tlA.0-1751568116448-0.0.1.1-604800000
+      host:
+      - api.openai.com
+      user-agent:
+      - OpenAI/Python 1.0.0
+      x-stainless-arch:
+      - arm64
+      x-stainless-lang:
+      - python
+      x-stainless-os:
+      - MacOS
+      x-stainless-package-version:
+      - 1.0.0
+      x-stainless-runtime:
+      - CPython
+      x-stainless-runtime-version:
+      - 3.10.13
+    method: POST
+    uri: https://api.openai.com/v1/chat/completions
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAAAwAAAP//dFTbjts2EH33Vwz4lCxsw3L3Br9tECRtEQTYbNoC7QbGmBxJk1IkOxwl
+        VYP994Ky19ZumxdB4JlzeObGbzMAw85swNgW1XbJL16ln2/wdX/+8fXt7c3b2zvp3r37dbj88Nvv
+        H/Ng5oURd5/J6iNraWOXPCnHsIetECoV1erqorq4vK6qyxHooiNfaE3SxXlcrFfr88XqerG6PBDb
+        yJay2cAfMwCAb+O3WAyO/jYbWM0fTzrKGRsym2MQgJHoy4nBnDkrBjXzE2hjUAqj6zuV3movHBpA
+        cKi4w0yQbUsdAtU1WeUv5AfgDFZ6y+ihjgIU8p5VOMBBqRHWYQ6JpI7SYbA0BwwOskWPO/aswxJ+
+        JCFAIcixI2gokKCHpmdHngNl0Agt+QRD7MFR5iYAgsRdn/W5vc19uA/VEs7OfgmOpKTp4AP91bNQ
+        R0Hz5uzsPgDAAt6itiSA3oO2BDIJGj32J4GCjzkVB4HIFUtZo9DyIPaTo6BcD2No+VWmDC8Ud57y
+        y1FPyGOZgtxyyrAj/UoUSny3LKbXxfT7UiXP/4yBJ6s3KfkBwhQE6f2+NOS544BaUnB9cBjsMF44
+        toOeNePR8J2iKHxlbUfLNUvWww2llx28qN6/2RtPEi2Rm06LUN1nLMMaeu8nAIYQdZ9mmdNPB+Th
+        OJk+NkniLj+jmpoD53YrhDmGMoWeQqOtGfGHGcCncQf6J2NtksQu6VbjnzReWK3XhyUwp7WbwKtH
+        VKOinwA/HHlPJLeOFNnnySIZi7YlN+FWF+tjGtg7jidsNZtk/19L/ye/rwCHZqLyXfkTYC0lJbdN
+        Qo7t07RPYULlafpe2LHWo2GTSb6wpa0ySemIoxp7v381TB6yUretOTQkSXj/dNRpu7q6vqpoje7a
+        zB5m/wIAAP//AwDhe2k9QwUAAA==
+    headers:
+      CF-RAY:
+      - 959885181e8fa4a0-EWR
+      Connection:
+      - keep-alive
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - application/json
+      Date:
+      - Thu, 03 Jul 2025 18:41:59 GMT
+      Server:
+      - cloudflare
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      access-control-expose-headers:
+      - X-Request-ID
+      alt-svc:
+      - h3=":443"; ma=86400
+      cf-cache-status:
+      - DYNAMIC
+      openai-organization:
+      - datadog-staging
+      openai-processing-ms:
+      - '2807'
+      openai-version:
+      - '2020-10-01'
+      strict-transport-security:
+      - max-age=31536000; includeSubDomains; preload
+      x-envoy-upstream-service-time:
+      - '2811'
+      x-ratelimit-limit-requests:
+      - '10000'
+      x-ratelimit-limit-tokens:
+      - '30000000'
+      x-ratelimit-remaining-requests:
+      - '9999'
+      x-ratelimit-remaining-tokens:
+      - '29998186'
+      x-ratelimit-reset-requests:
+      - 6ms
+      x-ratelimit-reset-tokens:
+      - 3ms
+      x-request-id:
+      - req_c0f374d27da52b49fbc9e28b67f5496a
+    status:
+      code: 200
+      message: OK
+version: 1