Baggage support (#4563)

packages/datadog-plugin-aws-sdk/test/eventbridge.spec.js

@@ -27,6 +27,7 @@ describe('EventBridge', () => {
             _traceFlags: {
               sampled: 1
             },
+            _baggageItems: {},
             'x-datadog-trace-id': traceId,
             'x-datadog-parent-id': parentId,
             'x-datadog-sampling-priority': '1',

packages/dd-trace/src/config.js

@@ -462,6 +462,9 @@ class Config {
     this._setValue(defaults, 'appsec.stackTrace.maxDepth', 32)
     this._setValue(defaults, 'appsec.stackTrace.maxStackTraces', 2)
     this._setValue(defaults, 'appsec.wafTimeout', 5e3) // µs
+    this._setValue(defaults, 'baggageMaxBytes', 8192)
+    this._setValue(defaults, 'baggageMaxItems', 64)
+    this._setValue(defaults, 'ciVisibilityTestSessionName', '')
     this._setValue(defaults, 'clientIpEnabled', false)
     this._setValue(defaults, 'clientIpHeader', null)
     this._setValue(defaults, 'codeOriginForSpans.enabled', false)
@@ -506,6 +509,7 @@ class Config {
     this._setValue(defaults, 'llmobs.mlApp', undefined)
     this._setValue(defaults, 'ciVisibilityTestSessionName', '')
     this._setValue(defaults, 'ciVisAgentlessLogSubmissionEnabled', false)
+    this._setValue(defaults, 'legacyBaggageEnabled', true)
     this._setValue(defaults, 'isTestDynamicInstrumentationEnabled', false)
     this._setValue(defaults, 'logInjection', false)
     this._setValue(defaults, 'lookup', undefined)
@@ -551,8 +555,8 @@ class Config {
     this._setValue(defaults, 'traceId128BitGenerationEnabled', true)
     this._setValue(defaults, 'traceId128BitLoggingEnabled', false)
     this._setValue(defaults, 'tracePropagationExtractFirst', false)
-    this._setValue(defaults, 'tracePropagationStyle.inject', ['datadog', 'tracecontext'])
-    this._setValue(defaults, 'tracePropagationStyle.extract', ['datadog', 'tracecontext'])
+    this._setValue(defaults, 'tracePropagationStyle.inject', ['datadog', 'tracecontext', 'baggage'])
+    this._setValue(defaults, 'tracePropagationStyle.extract', ['datadog', 'tracecontext', 'baggage'])
     this._setValue(defaults, 'tracePropagationStyle.otelPropagators', false)
     this._setValue(defaults, 'tracing', true)
     this._setValue(defaults, 'url', undefined)
@@ -637,6 +641,8 @@ class Config {
       DD_TRACE_AGENT_HOSTNAME,
       DD_TRACE_AGENT_PORT,
       DD_TRACE_AGENT_PROTOCOL_VERSION,
+      DD_TRACE_BAGGAGE_MAX_BYTES,
+      DD_TRACE_BAGGAGE_MAX_ITEMS,
       DD_TRACE_CLIENT_IP_ENABLED,
       DD_TRACE_CLIENT_IP_HEADER,
       DD_TRACE_ENABLED,
@@ -646,6 +652,7 @@ class Config {
       DD_TRACE_GIT_METADATA_ENABLED,
       DD_TRACE_GLOBAL_TAGS,
       DD_TRACE_HEADER_TAGS,
+      DD_TRACE_LEGACY_BAGGAGE_ENABLED,
       DD_TRACE_MEMCACHED_COMMAND_ENABLED,
       DD_TRACE_OBFUSCATION_QUERY_STRING_REGEXP,
       DD_TRACE_PARTIAL_FLUSH_MIN_SPANS,
@@ -717,6 +724,8 @@ class Config {
     this._envUnprocessed['appsec.stackTrace.maxStackTraces'] = DD_APPSEC_MAX_STACK_TRACES
     this._setValue(env, 'appsec.wafTimeout', maybeInt(DD_APPSEC_WAF_TIMEOUT))
     this._envUnprocessed['appsec.wafTimeout'] = DD_APPSEC_WAF_TIMEOUT
+    this._setValue(env, 'baggageMaxBytes', DD_TRACE_BAGGAGE_MAX_BYTES)
+    this._setValue(env, 'baggageMaxItems', DD_TRACE_BAGGAGE_MAX_ITEMS)
     this._setBoolean(env, 'clientIpEnabled', DD_TRACE_CLIENT_IP_ENABLED)
     this._setString(env, 'clientIpHeader', DD_TRACE_CLIENT_IP_HEADER)
     this._setBoolean(env, 'codeOriginForSpans.enabled', DD_CODE_ORIGIN_FOR_SPANS_ENABLED)
@@ -757,6 +766,7 @@ class Config {
     this._setArray(env, 'injectionEnabled', DD_INJECTION_ENABLED)
     this._setBoolean(env, 'isAzureFunction', getIsAzureFunction())
     this._setBoolean(env, 'isGCPFunction', getIsGCPFunction())
+    this._setBoolean(env, 'legacyBaggageEnabled', DD_TRACE_LEGACY_BAGGAGE_ENABLED)
     this._setBoolean(env, 'llmobs.agentlessEnabled', DD_LLMOBS_AGENTLESS_ENABLED)
     this._setBoolean(env, 'llmobs.enabled', DD_LLMOBS_ENABLED)
     this._setString(env, 'llmobs.mlApp', DD_LLMOBS_ML_APP)
@@ -893,6 +903,8 @@ class Config {
     this._optsUnprocessed['appsec.wafTimeout'] = options.appsec.wafTimeout
     this._setBoolean(opts, 'clientIpEnabled', options.clientIpEnabled)
     this._setString(opts, 'clientIpHeader', options.clientIpHeader)
+    this._setValue(opts, 'baggageMaxBytes', options.baggageMaxBytes)
+    this._setValue(opts, 'baggageMaxItems', options.baggageMaxItems)
     this._setBoolean(opts, 'codeOriginForSpans.enabled', options.codeOriginForSpans?.enabled)
     this._setString(opts, 'dbmPropagationMode', options.dbmPropagationMode)
     if (options.dogstatsd) {
@@ -930,6 +942,7 @@ class Config {
     }
     this._setString(opts, 'iast.telemetryVerbosity', options.iast && options.iast.telemetryVerbosity)
     this._setBoolean(opts, 'isCiVisibility', options.isCiVisibility)
+    this._setBoolean(opts, 'legacyBaggageEnabled', options.legacyBaggageEnabled)
     this._setBoolean(opts, 'llmobs.agentlessEnabled', options.llmobs?.agentlessEnabled)
     this._setString(opts, 'llmobs.mlApp', options.llmobs?.mlApp)
     this._setBoolean(opts, 'logInjection', options.logInjection)

packages/dd-trace/src/noop/span.js

@@ -16,6 +16,9 @@ class NoopSpan {
   setOperationName (name) { return this }
   setBaggageItem (key, value) { return this }
   getBaggageItem (key) {}
+  getAllBaggageItems () {}
+  removeBaggageItem (key) { return this }
+  removeAllBaggageItems () { return this }
   setTag (key, value) { return this }
   addTags (keyValueMap) { return this }
   addLink (link) { return this }

packages/dd-trace/src/opentracing/propagation/text_map.js

@@ -109,10 +109,42 @@ class TextMapPropagator {
     }
   }
 
+  _encodeOtelBaggageKey (key) {
+    let encoded = encodeURIComponent(key)
+    encoded = encoded.replaceAll('(', '%28')
+    encoded = encoded.replaceAll(')', '%29')
+    return encoded
+  }
+
   _injectBaggageItems (spanContext, carrier) {
-    spanContext._baggageItems && Object.keys(spanContext._baggageItems).forEach(key => {
-      carrier[baggagePrefix + key] = String(spanContext._baggageItems[key])
-    })
+    if (this._config.legacyBaggageEnabled) {
+      spanContext._baggageItems && Object.keys(spanContext._baggageItems).forEach(key => {
+        carrier[baggagePrefix + key] = String(spanContext._baggageItems[key])
+      })
+    }
+    if (this._hasPropagationStyle('inject', 'baggage')) {
+      if (this._config.baggageMaxItems < 1) return
+      let baggage = ''
+      let counter = 1
+      for (const [key, value] of Object.entries(spanContext._baggageItems)) {
+        baggage += `${this._encodeOtelBaggageKey(String(key).trim())}=${encodeURIComponent(String(value).trim())},`
+        if (counter === this._config.baggageMaxItems || counter > this._config.baggageMaxItems) break
+        counter += 1
+      }
+      baggage = baggage.slice(0, baggage.length - 1)
+      let buf = Buffer.from(baggage)
+      if (buf.length > this._config.baggageMaxBytes) {
+        const originalBaggages = baggage.split(',')
+        buf = buf.subarray(0, this._config.baggageMaxBytes)
+        const truncatedBaggages = buf.toString('utf8').split(',')
+        const lastPairIndex = truncatedBaggages.length - 1
+        if (truncatedBaggages[lastPairIndex] !== originalBaggages[lastPairIndex]) {
+          truncatedBaggages.splice(lastPairIndex, 1)
+        }
+        baggage = truncatedBaggages.slice(0, this._config.baggageMaxItems).join(',')
+      }
+      if (baggage) carrier.baggage = baggage
+    }
   }
 
   _injectTags (spanContext, carrier) {
@@ -301,6 +333,11 @@ class TextMapPropagator {
         default:
           log.warn(`Unknown propagation style: ${extractor}`)
       }
+
+      if (this._config.tracePropagationStyle.extract.includes('baggage') && carrier.baggage) {
+        spanContext = spanContext || new DatadogSpanContext()
+        this._extractBaggageItems(carrier, spanContext)
+      }
     }
 
     return spanContext || this._extractSqsdContext(carrier)
@@ -312,7 +349,7 @@ class TextMapPropagator {
     if (!spanContext) return spanContext
 
     this._extractOrigin(carrier, spanContext)
-    this._extractBaggageItems(carrier, spanContext)
+    this._extractLegacyBaggageItems(carrier, spanContext)
     this._extractSamplingPriority(carrier, spanContext)
     this._extractTags(carrier, spanContext)
 
@@ -446,7 +483,7 @@ class TextMapPropagator {
         }
       })
 
-      this._extractBaggageItems(carrier, spanContext)
+      this._extractLegacyBaggageItems(carrier, spanContext)
       return spanContext
     }
     return null
@@ -530,14 +567,43 @@ class TextMapPropagator {
     }
   }
 
-  _extractBaggageItems (carrier, spanContext) {
-    Object.keys(carrier).forEach(key => {
-      const match = key.match(baggageExpr)
+  _decodeOtelBaggageKey (key) {
+    let decoded = decodeURIComponent(key)
+    decoded = decoded.replaceAll('%28', '(')
+    decoded = decoded.replaceAll('%29', ')')
+    return decoded
+  }
 
-      if (match) {
-        spanContext._baggageItems[match[1]] = carrier[key]
+  _extractLegacyBaggageItems (carrier, spanContext) {
+    if (this._config.legacyBaggageEnabled) {
+      Object.keys(carrier).forEach(key => {
+        const match = key.match(baggageExpr)
+
+        if (match) {
+          spanContext._baggageItems[match[1]] = carrier[key]
+        }
+      })
+    }
+  }
+
+  _extractBaggageItems (carrier, spanContext) {
+    const baggages = carrier.baggage.split(',')
+    for (const keyValue of baggages) {
+      if (!keyValue.includes('=')) {
+        spanContext._baggageItems = {}
+        return
       }
-    })
+      let [key, value] = keyValue.split('=')
+      key = this._decodeOtelBaggageKey(key.trim())
+      value = decodeURIComponent(value.trim())
+      if (!key || !value) {
+        spanContext._baggageItems = {}
+        return
+      }
+      // the current code assumes precedence of ot-baggage- (legacy opentracing baggage) over baggage
+      if (key in spanContext._baggageItems) return
+      spanContext._baggageItems[key] = value
+    }
   }
 
   _extractSamplingPriority (carrier, spanContext) {

packages/dd-trace/src/opentracing/span.js

@@ -145,6 +145,18 @@ class DatadogSpan {
     return this._spanContext._baggageItems[key]
   }
 
+  getAllBaggageItems () {
+    return JSON.stringify(this._spanContext._baggageItems)
+  }
+
+  removeBaggageItem (key) {
+    delete this._spanContext._baggageItems[key]
+  }
+
+  removeAllBaggageItems () {
+    this._spanContext._baggageItems = {}
+  }
+
   setTag (key, value) {
     this._addTags({ [key]: value })
     return this

packages/dd-trace/test/config.spec.js

@@ -232,8 +232,8 @@ describe('Config', () => {
     expect(config).to.have.property('spanRemoveIntegrationFromService', false)
     expect(config).to.have.property('instrumentation_config_id', undefined)
     expect(config).to.have.deep.property('serviceMapping', {})
-    expect(config).to.have.nested.deep.property('tracePropagationStyle.inject', ['datadog', 'tracecontext'])
-    expect(config).to.have.nested.deep.property('tracePropagationStyle.extract', ['datadog', 'tracecontext'])
+    expect(config).to.have.nested.deep.property('tracePropagationStyle.inject', ['datadog', 'tracecontext', 'baggage'])
+    expect(config).to.have.nested.deep.property('tracePropagationStyle.extract', ['datadog', 'tracecontext', 'baggage'])
     expect(config).to.have.nested.property('experimental.runtimeId', false)
     expect(config).to.have.nested.property('experimental.exporter', undefined)
     expect(config).to.have.nested.property('experimental.enableGetRumData', false)

packages/dd-trace/test/opentracing/propagation/text_map.spec.js

@@ -46,7 +46,8 @@ describe('TextMapPropagator', () => {
     textMap = {
       'x-datadog-trace-id': '123',
       'x-datadog-parent-id': '456',
-      'ot-baggage-foo': 'bar'
+      'ot-baggage-foo': 'bar',
+      baggage: 'foo=bar'
     }
     baggageItems = {}
   })
@@ -77,25 +78,61 @@ describe('TextMapPropagator', () => {
       expect(carrier).to.have.property('x-datadog-trace-id', '123')
       expect(carrier).to.have.property('x-datadog-parent-id', '456')
       expect(carrier).to.have.property('ot-baggage-foo', 'bar')
+      expect(carrier).to.have.property('baggage', 'foo=bar')
     })
 
     it('should handle non-string values', () => {
       const carrier = {}
-      const spanContext = createContext({
-        baggageItems: {
-          number: 1.23,
-          bool: true,
-          array: ['foo', 'bar'],
-          object: {}
-        }
-      })
+      const baggageItems = {
+        number: 1.23,
+        bool: true,
+        array: ['foo', 'bar'],
+        object: {}
+      }
+      const spanContext = createContext({ baggageItems })
 
       propagator.inject(spanContext, carrier)
 
       expect(carrier['ot-baggage-number']).to.equal('1.23')
       expect(carrier['ot-baggage-bool']).to.equal('true')
       expect(carrier['ot-baggage-array']).to.equal('foo,bar')
       expect(carrier['ot-baggage-object']).to.equal('[object Object]')
+      expect(carrier.baggage).to.be.equal('number=1.23,bool=true,array=foo%2Cbar,object=%5Bobject%20Object%5D')
+    })
+
+    it('should handle special characters in baggage', () => {
+      const carrier = {}
+      const baggageItems = {
+        '",;\\()/:<=>?@[]{}': '",;\\'
+      }
+      const spanContext = createContext({ baggageItems })
+
+      propagator.inject(spanContext, carrier)
+      expect(carrier.baggage).to.be.equal('%22%2C%3B%5C%28%29%2F%3A%3C%3D%3E%3F%40%5B%5D%7B%7D=%22%2C%3B%5C')
+    })
+
+    it('should drop excess baggage items when there are too many pairs', () => {
+      const carrier = {}
+      const baggageItems = {}
+      for (let i = 0; i < config.baggageMaxItems + 1; i++) {
+        baggageItems[`key-${i}`] = i
+      }
+      const spanContext = createContext({ baggageItems })
+
+      propagator.inject(spanContext, carrier)
+      expect(carrier.baggage.split(',').length).to.equal(config.baggageMaxItems)
+    })
+
+    it('should drop excess baggage items when the resulting baggage header contains many bytes', () => {
+      const carrier = {}
+      const baggageItems = {
+        raccoon: 'chunky',
+        foo: Buffer.alloc(config.baggageMaxBytes).toString()
+      }
+      const spanContext = createContext({ baggageItems })
+
+      propagator.inject(spanContext, carrier)
+      expect(carrier.baggage).to.equal('raccoon=chunky')
     })
 
     it('should inject an existing sampling priority', () => {
@@ -363,9 +400,57 @@ describe('TextMapPropagator', () => {
       expect(spanContext.toTraceId()).to.equal(carrier['x-datadog-trace-id'])
       expect(spanContext.toSpanId()).to.equal(carrier['x-datadog-parent-id'])
       expect(spanContext._baggageItems.foo).to.equal(carrier['ot-baggage-foo'])
+      expect(spanContext._baggageItems).to.deep.equal({ foo: 'bar' })
       expect(spanContext._isRemote).to.equal(true)
     })
 
+    it('should extract otel baggage items with special characters', () => {
+      process.env.DD_TRACE_BAGGAGE_ENABLED = true
+      config = new Config()
+      propagator = new TextMapPropagator(config)
+      const carrier = {
+        'x-datadog-trace-id': '123',
+        'x-datadog-parent-id': '456',
+        baggage: '%22%2C%3B%5C%28%29%2F%3A%3C%3D%3E%3F%40%5B%5D%7B%7D=%22%2C%3B%5C'
+      }
+      const spanContext = propagator.extract(carrier)
+      expect(spanContext._baggageItems).to.deep.equal({ '",;\\()/:<=>?@[]{}': '",;\\' })
+    })
+
+    it('should not extract baggage when the header is malformed', () => {
+      const carrierA = {
+        'x-datadog-trace-id': '123',
+        'x-datadog-parent-id': '456',
+        baggage: 'no-equal-sign,foo=gets-dropped-because-previous-pair-is-malformed'
+      }
+      const spanContextA = propagator.extract(carrierA)
+      expect(spanContextA._baggageItems).to.deep.equal({})
+
+      const carrierB = {
+        'x-datadog-trace-id': '123',
+        'x-datadog-parent-id': '456',
+        baggage: 'foo=gets-dropped-because-subsequent-pair-is-malformed,='
+      }
+      const spanContextB = propagator.extract(carrierB)
+      expect(spanContextB._baggageItems).to.deep.equal({})
+
+      const carrierC = {
+        'x-datadog-trace-id': '123',
+        'x-datadog-parent-id': '456',
+        baggage: '=no-key'
+      }
+      const spanContextC = propagator.extract(carrierC)
+      expect(spanContextC._baggageItems).to.deep.equal({})
+
+      const carrierD = {
+        'x-datadog-trace-id': '123',
+        'x-datadog-parent-id': '456',
+        baggage: 'no-value='
+      }
+      const spanContextD = propagator.extract(carrierD)
+      expect(spanContextD._baggageItems).to.deep.equal({})
+    })
+
     it('should convert signed IDs to unsigned', () => {
       textMap['x-datadog-trace-id'] = '-123'
       textMap['x-datadog-parent-id'] = '-456'