feat(logging): auto enable log injection for structured loggers

.github/workflows/system-tests.yml

@@ -21,7 +21,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '3b3df04b753484a73a4b78fcdd95d5076da87602'
+          ref: '64afc5f8c1c2aa19adefdf1abf5559b62af8e784'
 
       - name: Build agent
         run: ./build.sh -i agent
@@ -69,7 +69,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '3b3df04b753484a73a4b78fcdd95d5076da87602'
+          ref: '64afc5f8c1c2aa19adefdf1abf5559b62af8e784'
 
       - name: Checkout dd-trace-py
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -123,7 +123,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '3b3df04b753484a73a4b78fcdd95d5076da87602'
+          ref: '64afc5f8c1c2aa19adefdf1abf5559b62af8e784'
 
       - name: Build runner
         uses: ./.github/actions/install_runner
@@ -310,7 +310,7 @@ jobs:
           persist-credentials: false
           repository: 'DataDog/system-tests'
           # Automatically managed, use scripts/update-system-tests-version to update
-          ref: '3b3df04b753484a73a4b78fcdd95d5076da87602'
+          ref: '64afc5f8c1c2aa19adefdf1abf5559b62af8e784'
       - name: Checkout dd-trace-py
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:

ddtrace/_logger.py

@@ -14,6 +14,15 @@
 DEFAULT_FILE_SIZE_BYTES = 15 << 20  # 15 MB
 
 
+class LogInjectionState(object):
+    # Log injection is disabled
+    DISABLED = "false"
+    # Log injection is enabled, but not yet configured
+    ENABLED = "true"
+    # Log injection is enabled and configured for structured logging
+    STRUCTURED = "structured"
+
+
 def configure_ddtrace_logger():
     # type: () -> None
     """Configures ddtrace log levels and file paths.
@@ -90,3 +99,27 @@ def _add_file_handler(
         logger.addHandler(ddtrace_file_handler)
         logger.debug("ddtrace logs will be routed to %s", log_path)
     return ddtrace_file_handler
+
+
+def set_log_formatting():
+    # type: () -> None
+    """Sets the log format for the ddtrace logger."""
+    ddtrace_logger = logging.getLogger("ddtrace")
+    for handler in ddtrace_logger.handlers:
+        handler.setFormatter(logging.Formatter(DD_LOG_FORMAT))
+
+
+def get_log_injection_state(raw_config: Optional[str]) -> str:
+    """Returns the current log injection state."""
+    if raw_config:
+        normalized = raw_config.lower().strip()
+        if normalized == LogInjectionState.STRUCTURED:
+            return LogInjectionState.STRUCTURED
+        elif normalized in ("true", "1"):
+            return LogInjectionState.ENABLED
+        elif normalized not in ("false", "0"):
+            logging.warning(
+                "Invalid log injection state '%s'. Expected 'true', 'false', or 'structured'. Defaulting to 'false'.",
+                normalized,
+            )
+    return LogInjectionState.DISABLED

ddtrace/_monkey.py

@@ -91,11 +91,10 @@
     # Ignore some web framework integrations that might be configured explicitly in code
     "falcon": True,
     "pyramid": True,
-    # Auto-enable logging if the environment variable DD_LOGS_INJECTION is true
-    "logbook": config._logs_injection,
-    "logging": config._logs_injection,
-    "loguru": config._logs_injection,
-    "structlog": config._logs_injection,
+    "logbook": True,
+    "logging": True,
+    "loguru": True,
+    "structlog": True,
     "pynamodb": True,
     "pyodbc": True,
     "fastapi": True,

ddtrace/_trace/product.py

@@ -46,17 +46,6 @@ def _on_global_config_update(cfg: Config, items: t.List[str]) -> None:
             if cfg._tracing_enabled is True and cfg._get_source("_tracing_enabled") != "remote_config":
                 tracer.enabled = True
 
-    if "_logs_injection" in items:
-        # TODO: Refactor the logs injection code to import from a core component
-        if config._logs_injection:
-            from ddtrace.contrib.internal.logging.patch import patch
-
-            patch()
-        else:
-            from ddtrace.contrib.internal.logging.patch import unpatch
-
-            unpatch()
-
 
 def post_preload():
     if _config.enabled:
@@ -139,7 +128,7 @@ def apm_tracing_rc(lib_config, dd_config):
             base_rc_config["_trace_sampling_rules"] = trace_sampling_rules
 
     if "log_injection_enabled" in lib_config:
-        base_rc_config["_logs_injection"] = lib_config["log_injection_enabled"]
+        base_rc_config["_logs_injection"] = str(lib_config["log_injection_enabled"]).lower()
 
     if "tracing_tags" in lib_config:
         tags = lib_config["tracing_tags"]

ddtrace/bootstrap/sitecustomize.py

@@ -20,24 +20,13 @@
 import warnings  # noqa:F401
 
 from ddtrace import config  # noqa:F401
-from ddtrace._logger import DD_LOG_FORMAT
 from ddtrace.internal.logger import get_logger  # noqa:F401
 from ddtrace.internal.module import ModuleWatchdog  # noqa:F401
 from ddtrace.internal.module import is_module_installed
 from ddtrace.internal.telemetry import telemetry_writer
 from ddtrace.internal.utils.formats import asbool  # noqa:F401
 
 
-# Debug mode from the tracer will do the same here, so only need to do this otherwise.
-if config._logs_injection:
-    from ddtrace import patch
-
-    patch(logging=True)
-    ddtrace_logger = logging.getLogger("ddtrace")
-    for handler in ddtrace_logger.handlers:
-        handler.setFormatter(logging.Formatter(DD_LOG_FORMAT))
-
-
 log = get_logger(__name__)
 
 

ddtrace/contrib/_logbook.py

@@ -16,7 +16,8 @@
 Patch ``logbook``
 ~~~~~~~~~~~~~~~~~~~
 
-If using :ref:`ddtrace-run<ddtracerun>` then set the environment variable ``DD_LOGS_INJECTION=true``.
+Logbook support is auto-enabled when :ref:`ddtrace-run<ddtracerun>` and a structured logging format  (ex: JSON) is used.
+To disable this integration, set the environment variable ``DD_LOGS_INJECTION=false``.
 
 Or use :func:`patch()<ddtrace.patch>` to manually enable the integration::
 

ddtrace/contrib/_loguru.py

@@ -16,7 +16,8 @@
 Patch ``loguru``
 ~~~~~~~~~~~~~~~~~~~
 
-If using :ref:`ddtrace-run<ddtracerun>` then set the environment variable ``DD_LOGS_INJECTION=true``.
+Loguru support is auto-enabled when :ref:`ddtrace-run<ddtracerun>` and a structured logging format  (ex: JSON) is used.
+To disable this integration, set the environment variable ``DD_LOGS_INJECTION=false``.
 
 Or use :func:`patch()<ddtrace.patch>` to manually enable the integration::
 

ddtrace/contrib/_structlog.py

@@ -15,7 +15,8 @@
 Patch ``structlog``
 ~~~~~~~~~~~~~~~~~~~
 
-If using :ref:`ddtrace-run<ddtracerun>` then set the environment variable ``DD_LOGS_INJECTION=true``.
+Structlog support is auto-enabled when :ref:`ddtrace-run<ddtracerun>` and a structured logging (ex: JSON) is used.
+To disable this integration, set the environment variable ``DD_LOGS_INJECTION=false``.
 
 Or use :func:`patch()<ddtrace.patch>` to manually enable the integration::
 

ddtrace/contrib/internal/logbook/patch.py

@@ -5,6 +5,7 @@
 
 import ddtrace
 from ddtrace import config
+from ddtrace._logger import LogInjectionState
 from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_ENV
 from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_SERVICE
 from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_SPAN_ID
@@ -46,8 +47,9 @@ def _tracer_injection(event_dict):
 
 def _w_process_record(func, instance, args, kwargs):
     # patch logger to include datadog info before logging
-    record = get_argument_value(args, kwargs, 0, "record")
-    _tracer_injection(record.extra)
+    if config._logs_injection != LogInjectionState.DISABLED:
+        record = get_argument_value(args, kwargs, 0, "record")
+        _tracer_injection(record.extra)
     return func(*args, **kwargs)
 
 

ddtrace/contrib/internal/logging/patch.py

@@ -5,6 +5,8 @@
 
 import ddtrace
 from ddtrace import config
+from ddtrace._logger import LogInjectionState
+from ddtrace._logger import set_log_formatting
 from ddtrace.contrib.internal.trace_utils import unwrap as _u
 from ddtrace.internal.utils import get_argument_value
 
@@ -73,6 +75,9 @@ def _get_tracer(tracer=None):
 def _w_makeRecord(func, instance, args, kwargs):
     # Get the LogRecord instance for this log
     record = func(*args, **kwargs)
+    if config._logs_injection != LogInjectionState.ENABLED:
+        # log injection is opt-in for non-structured logging
+        return record
 
     setattr(record, RECORD_ATTR_VERSION, config.version or RECORD_ATTR_VALUE_EMPTY)
     setattr(record, RECORD_ATTR_ENV, config.env or RECORD_ATTR_VALUE_EMPTY)
@@ -98,6 +103,8 @@ def _w_makeRecord(func, instance, args, kwargs):
 
 
 def _w_StrFormatStyle_format(func, instance, args, kwargs):
+    if config._logs_injection != LogInjectionState.ENABLED:
+        return func(*args, **kwargs)
     # The format string "dd.service={dd.service}" expects
     # the record to have a "dd" property which is an object that
     # has a "service" property
@@ -139,6 +146,13 @@ def patch():
         else:
             _w(logging.StrFormatStyle, "format", _w_StrFormatStyle_format)
 
+    if config._logs_injection == LogInjectionState.ENABLED:
+        # Only set the formatter is DD_LOGS_INJECTION is set to True. We do not want to modify
+        # unstructured logs if a user has not enabled logs injection.
+        # Also, the Datadog log format must be set after the logging module has been patched,
+        # otherwise the formatter will raise an exception.
+        set_log_formatting()
+
 
 def unpatch():
     if getattr(logging, "_datadog_patch", False):

ddtrace/contrib/internal/loguru/patch.py

@@ -5,6 +5,7 @@
 
 import ddtrace
 from ddtrace import config
+from ddtrace._logger import LogInjectionState
 from ddtrace.contrib.internal.trace_utils import unwrap as _u
 
 from ..logging.constants import RECORD_ATTR_ENV
@@ -31,6 +32,9 @@ def _supported_versions() -> Dict[str, str]:
 
 
 def _tracer_injection(event_dict):
+    if config._logs_injection == LogInjectionState.DISABLED:
+        # log injection is opt-out for structured logging
+        return event_dict
     trace_details = ddtrace.tracer.get_log_correlation_context()
 
     event_dd_attributes = {}

ddtrace/contrib/internal/structlog/patch.py

@@ -4,6 +4,7 @@
 
 import ddtrace
 from ddtrace import config
+from ddtrace._logger import LogInjectionState
 from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_ENV
 from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_SERVICE
 from ddtrace.contrib.internal.logging.constants import RECORD_ATTR_SPAN_ID
@@ -32,6 +33,9 @@ def _supported_versions() -> Dict[str, str]:
 
 
 def _tracer_injection(_, __, event_dict):
+    if config._logs_injection == LogInjectionState.DISABLED:
+        return event_dict
+
     trace_details = ddtrace.tracer.get_log_correlation_context()
 
     # add ids to structlog event dictionary

ddtrace/settings/_config.py

@@ -18,6 +18,8 @@
 from ddtrace.internal.telemetry import validate_otel_envs
 from ddtrace.internal.utils.cache import cachedmethod
 
+from .._logger import LogInjectionState
+from .._logger import get_log_injection_state
 from ..internal import gitmetadata
 from ..internal.constants import _PROPAGATION_BEHAVIOR_DEFAULT
 from ..internal.constants import _PROPAGATION_BEHAVIOR_IGNORE
@@ -361,9 +363,9 @@ def _default_config() -> Dict[str, _ConfigItem]:
             modifier=str,
         ),
         "_logs_injection": _ConfigItem(
-            default=False,
+            default=LogInjectionState.STRUCTURED,
             envs=["DD_LOGS_INJECTION"],
-            modifier=asbool,
+            modifier=get_log_injection_state,
         ),
         "_trace_http_header_tags": _ConfigItem(
             default=lambda: {},

docs/configuration.rst

@@ -741,9 +741,9 @@ Logs
 .. ddtrace-configuration-options::
 
    DD_LOGS_INJECTION:
-     type: Boolean
-     default: False
-     description: Enables :ref:`Logs Injection`.
+     type: string
+     default: structured
+     description: Enables :ref:`Logs Injection`. Supported values are ``false``, ``true``, and ``structured``.
 
    DD_TRACE_DEBUG:
      type: Boolean

releasenotes/notes/enable-structored-logging-by-default-b6ad86e55b3582be.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    loguru,structlog,logbook: Enable trace-log correlation for structured loggers by default.
+  - |
+    loguru,structlog,logbook: Adds support for trace-log correlation via remote configuration. Previously, this functionality was only available for Python’s built-in logging library.

tests/commands/test_runner.py

@@ -330,7 +330,7 @@ def test_info_no_configs():
         b"Application Security enabled: False",
         b"Remote Configuration enabled: False",
         b"Debug logging: False",
-        b"Log injection enabled: False",
+        b"Log injection enabled: structured",
         b"Health metrics enabled: False",
         b"Partial flushing enabled: True",
         b"Partial flush minimum number of spans: 300",
@@ -386,7 +386,7 @@ def test_info_w_configs():
         b"Remote Configuration enabled: True",
         b"IAST enabled (experimental)",
         b"Debug logging: True",
-        b"Log injection enabled: True",
+        b"Log injection enabled: true",
         b"Health metrics enabled: False",
         b"Partial flushing enabled: True",
         b"Partial flush minimum number of spans: 1000",