Merge branch 'main' into main

Author: brettlangdon

.github/workflows/system-tests.yml

@@ -42,7 +42,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: 'e317348e48d9e934bb0f743e1537126447199dc4'
+          ref: 'dacdb8249761abc576efde661345c23ac8731beb'
 
       - name: Checkout dd-trace-py
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -96,7 +96,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: 'e317348e48d9e934bb0f743e1537126447199dc4'
+          ref: 'dacdb8249761abc576efde661345c23ac8731beb'
 
       - name: Build runner
         uses: ./.github/actions/install_runner
@@ -277,7 +277,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: 'e317348e48d9e934bb0f743e1537126447199dc4'
+          ref: 'dacdb8249761abc576efde661345c23ac8731beb'
       - name: Checkout dd-trace-py
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:

.gitlab/templates/debugging/exploration.yml

@@ -1,25 +1,52 @@
-"debugging/exploration/boto3":
+".debugging/exploration":
   stage: debugging
   extends: .cached_testrunner
   timeout: 30m
   variables:
-    DD_DEBUGGER_EXPL_INCLUDE: "boto3"
     DD_DEBUGGER_EXPL_OUTPUT_FILE: "${{CI_PROJECT_DIR}}/debugger-expl.txt"
     DD_DEBUGGER_EXPL_STATUS_MESSAGES: 1
     PYTEST_PLUGINS: "exploration"
     PYTHONPATH: "${{CI_PROJECT_DIR}}/tests/debugging/exploration/pytest"
+  before_script:
+    - !reference [.cached_testrunner, before_script]
+    - python${{PYTHON_VERSION}} -m pip install -e .
+  after_script:
+    - cat ${{DD_DEBUGGER_EXPL_OUTPUT_FILE}}
+    - !reference [.cached_testrunner, after_script]
+  needs: []
+  artifacts:
+    paths:
+      - ${{DD_DEBUGGER_EXPL_OUTPUT_FILE}}
+
+"debugging/exploration/boto3":
+  extends: ".debugging/exploration"
+  variables:
+    DD_DEBUGGER_EXPL_INCLUDE: "boto3"
+  parallel:
+    matrix:
+      - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        BOTO3_TAG: 1.38.44
+  script: |
+    git clone --depth 1 --branch ${{BOTO3_TAG}} https://github.com/boto/boto3.git
+    cd boto3
+    python${{PYTHON_VERSION}} scripts/ci/install
+    python${{PYTHON_VERSION}} scripts/ci/run-tests --test-runner 'pytest -svv -W error -W "ignore::dateutil.parser._parser.UnknownTimezoneWarning" -W "ignore::DeprecationWarning"'
+
+"debugging/exploration/botocore":
+  extends: ".debugging/exploration"
+  timeout: 60m
+  variables:
+    DD_DEBUGGER_EXPL_INCLUDE: "botocore"
+    DD_DEBUGGER_EXPL_PROFILER_DELETE_FUNCTION_PROBES: 1
+    DD_DEBUGGER_EXPL_PROFILER_INSTRUMENTATION_RATE: 0.05
+    DD_DEBUGGER_EXPL_COVERAGE_DELETE_LINE_PROBES: 1
+    DD_DEBUGGER_EXPL_COVERAGE_INSTRUMENTATION_RATE: 0.05
   parallel:
     matrix:
       - PYTHON_VERSION: ["3.9", "3.10", "3.11", "3.12", "3.13"]
         BOTO3_TAG: 1.38.44
   script: |
-    python${{PYTHON_VERSION}} -m pip install -e .
     git clone --depth 1 --branch ${{BOTO3_TAG}} https://github.com/boto/boto3.git
     cd boto3
     python${{PYTHON_VERSION}} scripts/ci/install
     python${{PYTHON_VERSION}} scripts/ci/run-tests --test-runner 'pytest -svv -W error -W "ignore::dateutil.parser._parser.UnknownTimezoneWarning" -W "ignore::DeprecationWarning"'
-    cat ${{DD_DEBUGGER_EXPL_OUTPUT_FILE}}
-  needs: []
-  artifacts:
-    paths:
-      - ${{DD_DEBUGGER_EXPL_OUTPUT_FILE}}

ddtrace/debugging/_products/code_origin/span.py

@@ -1,3 +1,5 @@
+import enum
+
 from ddtrace.internal.products import manager as product_manager
 from ddtrace.settings._core import ValueSource
 from ddtrace.settings.code_origin import config
@@ -14,6 +16,12 @@ def post_preload():
     pass
 
 
+def _start():
+    from ddtrace.debugging._origin.span import SpanCodeOriginProcessorEntry
+
+    SpanCodeOriginProcessorEntry.enable()
+
+
 def start():
     if config.span.enabled:
         from ddtrace.debugging._origin.span import SpanCodeOriginProcessorEntry
@@ -33,6 +41,12 @@ def restart(join=False):
     pass
 
 
+def _stop():
+    from ddtrace.debugging._origin.span import SpanCodeOriginProcessorEntry
+
+    SpanCodeOriginProcessorEntry.disable()
+
+
 def stop(join=False):
     if config.span.enabled:
         from ddtrace.debugging._origin.span import SpanCodeOriginProcessorEntry
@@ -48,3 +62,13 @@ def stop(join=False):
 
 def at_exit(join=False):
     stop(join=join)
+
+
+class APMCapabilities(enum.IntFlag):
+    APM_TRACING_ENABLE_CODE_ORIGIN = 1 << 40
+
+
+def apm_tracing_rc(lib_config, _config):
+    if (enabled := lib_config.get("code_origin_enabled")) is not None:
+        should_start = (config.span.spec.enabled.full_name not in config.source or config.span.enabled) and enabled
+        _start() if should_start else _stop()

ddtrace/llmobs/_integrations/langchain.py

@@ -499,6 +499,8 @@ def _llmobs_set_tags_from_chat_model(
                 # do not append to the count, just set it once
                 if not is_workflow and not tokens_set_top_level:
                     tokens, run_id = self.check_token_usage_ai_message(chat_completion_msg)
+                    if run_id is None:
+                        continue
                     input_tokens, output_tokens, total_tokens = tokens
                     tokens_per_choice_run_id[run_id]["input_tokens"] = input_tokens
                     tokens_per_choice_run_id[run_id]["output_tokens"] = output_tokens
@@ -726,7 +728,7 @@ def check_token_usage_chat_or_llm_result(self, result):
 
         return input_tokens, output_tokens, total_tokens
 
-    def check_token_usage_ai_message(self, ai_message):
+    def check_token_usage_ai_message(self, ai_message) -> Tuple[Tuple[int, int, int], Optional[str]]:
         """Checks for token usage on an AI message object"""
         # depending on the provider + langchain-core version, the usage metadata can be in different places
         # either chat_completion_msg.usage_metadata or chat_completion_msg.response_metadata.{token}_usage
@@ -735,10 +737,10 @@ def check_token_usage_ai_message(self, ai_message):
         run_id = getattr(ai_message, "id", None) or getattr(ai_message, "run_id", "")
         run_id_base = "-".join(run_id.split("-")[:-1]) if run_id else ""
 
-        response_metadata = getattr(ai_message, "response_metadata", {}) or {}
-        usage = usage or response_metadata.get("usage", {}) or response_metadata.get("token_usage", {})
+        response_metadata = getattr(ai_message, "response_metadata", {})
+        usage = usage or response_metadata.get("usage") or response_metadata.get("token_usage")
         if usage is None or not isinstance(usage, dict):  # in case it is explicitly set to None
-            return 0, 0, 0
+            return (0, 0, 0), run_id_base
 
         # could either be "{prompt,completion}_tokens" or "{input,output}_tokens"
         input_tokens = usage.get("input_tokens", 0) or usage.get("prompt_tokens", 0)

releasenotes/notes/feat-co-in-product-enablement-6b70fc52d62721fa.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    code origin: added support for in-product enablement.

releasenotes/notes/langchain-llmobs-token-value-error-e157dce357408791.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    LLM Observability: Fixed an issue where grabbing token values for some providers through ``langchain`` libraries raised a ``ValueError``.

scripts/gen_gitlab_config.py

@@ -317,6 +317,19 @@ def gen_debugger_exploration() -> None:
     We need to generate this dynamically from a template because it depends
     on the cached testrunner job, which is also generated dynamically.
     """
+    from needs_testrun import pr_matches_patterns
+
+    if not pr_matches_patterns(
+        {
+            ".gitlab/templates/debugging/exploration.yml",
+            "ddtrace/debugging/*",
+            "ddtrace/internal/bytecode_injection/__init__.py",
+            "ddtrace/internal/wrapping/context.py",
+            "tests/debugging/exploration/*",
+        }
+    ):
+        return
+
     with TESTS_GEN.open("a") as f:
         f.write(template("debugging/exploration"))
 

tests/contrib/langchain/conftest.py

@@ -250,6 +250,7 @@ def _openai_completion_object(
 def _openai_chat_completion_object(
     n: int = 1,
     tools: bool = False,
+    include_usage: bool = True,
 ):
     from datetime import datetime
 
@@ -274,11 +275,6 @@ def _openai_chat_completion_object(
         object="chat.completion",
         choices=choices,
         created=int(datetime.now().timestamp()),
-        usage=CompletionUsage(
-            prompt_tokens=5,
-            completion_tokens=5,
-            total_tokens=10,
-        ),
     )
 
     if tools:
@@ -297,6 +293,13 @@ def _openai_chat_completion_object(
         for choice in completion.choices:
             choice.message.tool_calls = [tool_call]
 
+    if include_usage:
+        completion.usage = CompletionUsage(
+            prompt_tokens=5,
+            completion_tokens=5,
+            total_tokens=10,
+        )
+
     return completion
 
 
@@ -352,6 +355,18 @@ def openai_chat_completion(respx_mock):
     )
 
 
+@pytest.fixture
+@pytest.mark.respx()
+def openai_chat_completion_no_usage(respx_mock):
+    import httpx
+
+    completion = _openai_chat_completion_object(n=2, include_usage=False)
+
+    respx_mock.post("/v1/chat/completions").mock(
+        return_value=httpx.Response(200, json=completion.model_dump(mode="json"))
+    )
+
+
 @pytest.fixture
 @pytest.mark.respx()
 def openai_chat_completion_multiple(respx_mock):

tests/contrib/langchain/test_langchain_llmobs.py

@@ -130,6 +130,16 @@ def test_llmobs_openai_chat_model(langchain_openai, llmobs_events, tracer, opena
     )
 
 
+def test_llmobs_openai_chat_model_no_usage(langchain_openai, llmobs_events, tracer, openai_chat_completion_no_usage):
+    chat_model = langchain_openai.ChatOpenAI(temperature=0, max_tokens=256)
+    chat_model.invoke([HumanMessage(content="When do you use 'who' instead of 'whom'?")])
+
+    assert len(llmobs_events) == 1
+    assert llmobs_events[0]["metrics"].get("input_tokens") is None
+    assert llmobs_events[0]["metrics"].get("output_tokens") is None
+    assert llmobs_events[0]["metrics"].get("total_tokens") is None
+
+
 @mock.patch("langchain_core.language_models.chat_models.BaseChatModel._generate_with_cache")
 def test_llmobs_openai_chat_model_proxy(mock_generate, langchain_openai, llmobs_events, tracer, openai_chat_completion):
     mock_generate.return_value = mock_langchain_chat_generate_response

tests/debugging/exploration/_config.py

@@ -101,6 +101,13 @@ class ProfilerConfig(DDConfig):
             help="Whether to delete function probes after they are triggered",
         )
 
+        instrumentation_rate = DDConfig.v(
+            float,
+            "instrumentation_rate",
+            default=1.0,
+            help="Rate at which to instrument functions for profiling",
+        )
+
     class CoverageConfig(DDConfig):
         __item__ = "coverage"
         __prefix__ = "dd.debugger.expl.coverage"
@@ -119,5 +126,12 @@ class CoverageConfig(DDConfig):
             help="Whether to delete line probes after they are triggered",
         )
 
+        instrumentation_rate = DDConfig.v(
+            float,
+            "instrumentation_rate",
+            default=1.0,
+            help="Rate at which to instrument lines for coverage",
+        )
+
 
 config = ExplorationConfig()