fix(openfeature): harden agentless EVP fallback

packages/dd-trace/src/agent/info.js

@@ -17,7 +17,7 @@ module.exports = {
  * Fetches agent information from the /info endpoint
  * @param {URL} url - The agent URL
  * @param {Function} callback - Callback function with signature (err, agentInfo)
- * @param {{ deadline?: number, signal?: AbortSignal }} [options] - Request finalization options
+ * @param {{ deadline?: number, signal?: AbortSignal, retry?: boolean }} [options] - Request finalization options
  * @param {Function} [makeRequest] - Request implementation
  */
 function fetchAgentInfo (url, callback, options = {}, makeRequest = request) {

packages/dd-trace/src/evp_proxy/discovery.js

@@ -106,11 +106,12 @@ function selectEVPProxyPath (agentInfo, { supportedPaths, requiredHeaders = [] }
  * @param {string[]} options.supportedPaths - Supported paths in preference order
  * @param {string[]} [options.requiredHeaders] - Headers that the proxy must forward unchanged to intake. Each
  * header must appear in `evp_proxy_allowed_headers`. Do not include routing headers that the Agent consumes.
+ * @param {boolean} [options.retry] - Whether the Agent information request retries connection failures
  * @param {(error: Error|null, route?: {url: URL, basePath: string}) => void} callback - Result callback
  * @returns {void}
  */
 function discoverEVPProxy (url, options, callback) {
-  fetchAgentInfo(url, (error, agentInfo) => {
+  const handleAgentInfo = (error, agentInfo) => {
     if (error) {
       callback(error)
       return
@@ -124,7 +125,14 @@ function discoverEVPProxy (url, options, callback) {
 
     log.debug('EVP proxy route %s discovered through the configured local receiver', basePath)
     callback(null, { url, basePath })
-  })
+  }
+
+  if (options.retry === undefined) {
+    fetchAgentInfo(url, handleAgentInfo)
+    return
+  }
+
+  fetchAgentInfo(url, handleAgentInfo, { retry: options.retry })
 }
 
 module.exports = {

packages/dd-trace/src/openfeature/flagging_provider.js

@@ -10,7 +10,7 @@ const EvalMetricsHook = require('./eval-metrics-hook')
 const SpanEnrichmentHook = require('./span-enrichment-hook')
 const FlagEvalEVPHook = require('./writers/flag-eval-evp-hook')
 const FlagEvaluationsWriter = require('./writers/flag-evaluations')
-const { setExposureDeliveryStrategy } = require('./writers/util')
+const { setEventDeliveryStrategy } = require('./writers/util')
 
 /**
  * OpenFeature provider that integrates with Datadog's feature flagging system.
@@ -42,7 +42,7 @@ class FlaggingProvider extends DatadogNodeServerProvider {
       this.#flagEvalEVPWriter = new FlagEvaluationsWriter(config)
       const writer = this.#flagEvalEVPWriter
       this.hooks.push(new FlagEvalEVPHook(writer))
-      setExposureDeliveryStrategy(config, (enabled, route) => {
+      setEventDeliveryStrategy(config, (enabled, route) => {
         if (this.#flagEvalEVPWriter !== writer) return
         writer.setEnabled(enabled, route)
       })

packages/dd-trace/src/openfeature/index.js

@@ -3,7 +3,7 @@
 const { channel } = require('dc-polyfill')
 const log = require('../log')
 const ExposuresWriter = require('./writers/exposures')
-const { setExposureDeliveryStrategy } = require('./writers/util')
+const { setEventDeliveryStrategy } = require('./writers/util')
 
 const exposureSubmitCh = channel('ffe:exposure:submit')
 const flushCh = channel('ffe:writers:flush')
@@ -45,7 +45,7 @@ function enable (config) {
   exposureSubmitCh.subscribe(_handleExposureSubmit)
   flushCh.subscribe(_handleFlush)
 
-  setExposureDeliveryStrategy(config, (enabled, route) => {
+  setEventDeliveryStrategy(config, (enabled, route) => {
     if (exposuresWriter !== writer) return
 
     writer.setEnabled(enabled, route)

packages/dd-trace/src/openfeature/writers/base.js

@@ -292,14 +292,14 @@ class BaseFFEWriter {
 
       if (fallbackRoute && isAmbiguousNetworkFailure(error)) {
         log.debug(
-          '%s retrying through direct intake and switching future batches from %s%s after ambiguous failure',
+          '%s switching future batches from %s%s to direct intake after ambiguous failure without replaying the batch',
           this.constructor.name,
           route.url.href,
           route.endpoint
         )
         this.#activateRoute(fallbackRoute)
         this._fallbackRoute = undefined
-        this.#sendRequest(payload, eventCount, fallbackRoute)
+        log.error('Failed to send events to %s%s: %s', route.url.href, route.endpoint, error.message)
         return
       }
 

packages/dd-trace/src/openfeature/writers/util.js

@@ -11,17 +11,18 @@ const { discoverEVPProxy } = require('../../evp_proxy/discovery')
 const logger = require('../../log')
 
 let missingRouteWarningLogged = false
+const deliveryStrategyStates = new WeakMap()
 
 /**
- * Logs the unavailable exposure-delivery warning once.
+ * Logs the unavailable event-delivery warning once.
  *
  * @returns {void}
  */
-function warnExposureDeliveryUnavailable () {
+function warnEventDeliveryUnavailable () {
   if (missingRouteWarningLogged) return
   missingRouteWarningLogged = true
   logger.warn(
-    'Feature Flags exposure delivery is disabled because no compatible local EVP route or direct intake ' +
+    'Feature Flags event delivery is disabled because no compatible local EVP route or direct intake ' +
     'credentials are available.'
   )
 }
@@ -67,6 +68,7 @@ function setAgentlessStrategy (config, setWriterEnabledValue) {
   const directRoute = createDirectEVPRoute(config, EVP_EVENT_PLATFORM_SUBDOMAIN)
 
   discoverEVPProxy(config.url, {
+    retry: false,
     supportedPaths: [EVP_PROXY_PATH_V4, EVP_PROXY_PATH_V2],
   }, (error, localRoute) => {
     if (localRoute) {
@@ -95,25 +97,50 @@ function setAgentlessStrategy (config, setWriterEnabledValue) {
     if (error) {
       logger.debug('FFE Writer disabled - error getting local receiver info: %s', error.message)
     }
-    warnExposureDeliveryUnavailable()
+    warnEventDeliveryUnavailable()
     setWriterEnabledValue(false)
   })
 }
 
 /**
- * Applies the exposure-delivery strategy for the configured Feature Flags source.
+ * Applies one shared event-delivery strategy for the configured Feature Flags source.
  *
  * @param {import('../../config')} config - Tracer configuration object
  * @param {Function} setWriterEnabledValue - Callback to set the writer enabled state
  * @returns {void}
  */
-function setExposureDeliveryStrategy (config, setWriterEnabledValue) {
+function setEventDeliveryStrategy (config, setWriterEnabledValue) {
+  const existingState = deliveryStrategyStates.get(config)
+  if (existingState) {
+    if (existingState.result) {
+      setWriterEnabledValue(...existingState.result)
+    } else {
+      existingState.callbacks.push(setWriterEnabledValue)
+    }
+    return
+  }
+
+  const state = {
+    callbacks: [setWriterEnabledValue],
+    result: undefined,
+  }
+  deliveryStrategyStates.set(config, state)
+
+  const resolve = (...result) => {
+    state.result = result
+    const callbacks = state.callbacks
+    state.callbacks = []
+    for (const callback of callbacks) {
+      callback(...result)
+    }
+  }
+
   if (config.featureFlags?.DD_FEATURE_FLAGS_CONFIGURATION_SOURCE === 'agentless') {
-    setAgentlessStrategy(config, setWriterEnabledValue)
+    setAgentlessStrategy(config, resolve)
     return
   }
 
-  setAgentStrategy(config, setWriterEnabledValue)
+  setAgentStrategy(config, resolve)
 }
 
-module.exports = { setExposureDeliveryStrategy }
+module.exports = { setEventDeliveryStrategy }

packages/dd-trace/test/evp_proxy/discovery.spec.js

@@ -120,6 +120,17 @@ describe('EVP proxy discovery', () => {
       })
     })
 
+    it('passes an explicit retry policy to the Agent information request', (done) => {
+      fetchAgentInfo.yields(null, { endpoints: ['/evp_proxy/v2'] })
+
+      discoverEVPProxy(url, { ...options, retry: false }, (error, route) => {
+        assert.ifError(error)
+        assert.strictEqual(route.basePath, '/evp_proxy/v2')
+        sinon.assert.calledOnceWithExactly(fetchAgentInfo, url, sinon.match.func, { retry: false })
+        done()
+      })
+    })
+
     it('returns information request errors', (done) => {
       const expectedError = new Error('Agent unavailable')
       fetchAgentInfo.yields(expectedError)

packages/dd-trace/test/openfeature/flagging_provider.spec.js

@@ -25,7 +25,7 @@ describe('FlaggingProvider', () => {
   let mockFlagEvalWriterClass
   let mockFlagEvalEVPHook
   let mockFlagEvalEVPHookClass
-  let setExposureDeliveryStrategyStub
+  let setEventDeliveryStrategyStub
 
   beforeEach(() => {
     mockTracer = {
@@ -85,7 +85,7 @@ describe('FlaggingProvider', () => {
     mockFlagEvalEVPHook = {}
     mockFlagEvalEVPHookClass = sinon.stub().returns(mockFlagEvalEVPHook)
 
-    setExposureDeliveryStrategyStub = sinon.stub()
+    setEventDeliveryStrategyStub = sinon.stub()
 
     FlaggingProvider = proxyquire('../../src/openfeature/flagging_provider', {
       'dc-polyfill': {
@@ -97,7 +97,7 @@ describe('FlaggingProvider', () => {
       './span-enrichment-hook': mockSpanEnrichmentHookClass,
       './writers/flag-evaluations': mockFlagEvalWriterClass,
       './writers/flag-eval-evp-hook': mockFlagEvalEVPHookClass,
-      './writers/util': { setExposureDeliveryStrategy: setExposureDeliveryStrategyStub },
+      './writers/util': { setEventDeliveryStrategy: setEventDeliveryStrategyStub },
       '../../../../vendor/dist/@datadog/openfeature-node-server': { DatadogNodeServerProvider },
     })
   })
@@ -169,17 +169,17 @@ describe('FlaggingProvider', () => {
       assert.ok(!provider.hooks.includes(mockFlagEvalEVPHook),
         'EVP hook must not be registered when killswitch is false')
       sinon.assert.notCalled(mockFlagEvalWriterClass)
-      sinon.assert.notCalled(setExposureDeliveryStrategyStub)
-      assert.ok(!setExposureDeliveryStrategyStub.called,
+      sinon.assert.notCalled(setEventDeliveryStrategyStub)
+      assert.ok(!setEventDeliveryStrategyStub.called,
         'agent probe must not run when the killswitch disables the writer')
     })
 
     it('applies the selected EVP delivery route', () => {
       new FlaggingProvider(mockTracer, mockConfig) // eslint-disable-line no-new
 
-      sinon.assert.calledOnce(setExposureDeliveryStrategyStub)
+      sinon.assert.calledOnce(setEventDeliveryStrategyStub)
       const setEnabled = mockFlagEvalWriter.setEnabled
-      const selectRoute = setExposureDeliveryStrategyStub.firstCall.args[1]
+      const selectRoute = setEventDeliveryStrategyStub.firstCall.args[1]
       const route = {
         url: new URL('https://event-platform-intake.datadoghq.com'),
         basePath: '',

packages/dd-trace/test/openfeature/index.spec.js

@@ -14,7 +14,7 @@ describe('OpenFeature Module', () => {
   let openfeatureModule
   let mockWriter
   let ExposuresWriterStub
-  let setExposureDeliveryStrategyStub
+  let setEventDeliveryStrategyStub
 
   beforeEach(() => {
     config = {
@@ -30,11 +30,11 @@ describe('OpenFeature Module', () => {
     }
 
     ExposuresWriterStub = sinon.stub().returns(mockWriter)
-    setExposureDeliveryStrategyStub = sinon.stub()
+    setEventDeliveryStrategyStub = sinon.stub()
 
     openfeatureModule = proxyquire('../../src/openfeature', {
       './writers/exposures': ExposuresWriterStub,
-      './writers/util': { setExposureDeliveryStrategy: setExposureDeliveryStrategyStub },
+      './writers/util': { setEventDeliveryStrategy: setEventDeliveryStrategyStub },
     })
   })
 
@@ -52,12 +52,12 @@ describe('OpenFeature Module', () => {
       openfeatureModule.enable(config)
 
       sinon.assert.calledOnceWithExactly(ExposuresWriterStub, config)
-      sinon.assert.calledOnce(setExposureDeliveryStrategyStub)
+      sinon.assert.calledOnce(setEventDeliveryStrategyStub)
     })
 
     it('configures the writer with the selected exposure route', () => {
       openfeatureModule.enable(config)
-      const setWriterEnabled = setExposureDeliveryStrategyStub.firstCall.args[1]
+      const setWriterEnabled = setEventDeliveryStrategyStub.firstCall.args[1]
       const route = {
         url: new URL('http://serverless-init:8126'),
         basePath: '/evp_proxy/v4',
@@ -85,10 +85,10 @@ describe('OpenFeature Module', () => {
       ExposuresWriterStub.onSecondCall().returns(replacementWriter)
 
       openfeatureModule.enable(config)
-      const staleCallback = setExposureDeliveryStrategyStub.firstCall.args[1]
+      const staleCallback = setEventDeliveryStrategyStub.firstCall.args[1]
       openfeatureModule.disable()
       openfeatureModule.enable(config)
-      const currentCallback = setExposureDeliveryStrategyStub.secondCall.args[1]
+      const currentCallback = setEventDeliveryStrategyStub.secondCall.args[1]
 
       staleCallback(true, staleRoute)
       sinon.assert.notCalled(replacementWriter.setEnabled)

packages/dd-trace/test/openfeature/writers/exposures-transport.spec.js

@@ -8,7 +8,7 @@ const nock = require('nock')
 require('../../setup/core')
 const { clearCache } = require('../../../src/agent/info')
 const ExposuresWriter = require('../../../src/openfeature/writers/exposures')
-const { setExposureDeliveryStrategy } = require('../../../src/openfeature/writers/util')
+const { setEventDeliveryStrategy } = require('../../../src/openfeature/writers/util')
 
 describe('OpenFeature Exposures Writer transport', () => {
   let writer
@@ -40,6 +40,7 @@ describe('OpenFeature Exposures Writer transport', () => {
           'content-type': 'application/json',
           'x-datadog-evp-subdomain': 'event-platform-intake',
         },
+        badheaders: ['dd-api-key'],
       })
         .post('/evp_proxy/v4/api/v2/exposures')
         .reply(202, (uri, body) => {
@@ -58,7 +59,7 @@ describe('OpenFeature Exposures Writer transport', () => {
 
     writer = new ExposuresWriter(config)
     await new Promise((resolve, reject) => {
-      setExposureDeliveryStrategy(config, (enabled, route) => {
+      setEventDeliveryStrategy(config, (enabled, route) => {
         try {
           assert.strictEqual(enabled, true)
           assert.strictEqual(route.basePath, '/evp_proxy/v4')
@@ -114,7 +115,7 @@ describe('OpenFeature Exposures Writer transport', () => {
 
     writer = new ExposuresWriter(config)
     await new Promise((resolve, reject) => {
-      setExposureDeliveryStrategy(config, (enabled, route) => {
+      setEventDeliveryStrategy(config, (enabled, route) => {
         try {
           assert.strictEqual(enabled, true)
           assert.strictEqual(route.basePath, '/evp_proxy/v4')
@@ -169,7 +170,7 @@ describe('OpenFeature Exposures Writer transport', () => {
 
     writer = new ExposuresWriter(config)
     await new Promise((resolve, reject) => {
-      setExposureDeliveryStrategy(config, (enabled, route) => {
+      setEventDeliveryStrategy(config, (enabled, route) => {
         try {
           assert.strictEqual(enabled, true)
           assert.strictEqual(route.basePath, '')

packages/dd-trace/test/openfeature/writers/exposures.spec.js

@@ -394,6 +394,7 @@ describe('OpenFeature Exposures Writer', () => {
       assert.match(options.path, /\/evp_proxy\/v2\//)
       assert.strictEqual(options.headers['Content-Type'], 'application/json')
       assert.strictEqual(options.headers['X-Datadog-EVP-Subdomain'], 'event-platform-intake')
+      assert.strictEqual(options.headers['DD-API-KEY'], undefined)
 
       const parsedPayload = JSON.parse(payload)
       assert.ok(
@@ -424,6 +425,7 @@ describe('OpenFeature Exposures Writer', () => {
       assert.strictEqual(options.url, url)
       assert.strictEqual(options.path, '/evp_proxy/v4/api/v2/exposures')
       assert.strictEqual(options.headers['X-Datadog-EVP-Subdomain'], 'event-platform-intake')
+      assert.strictEqual(options.headers['DD-API-KEY'], undefined)
     })
 
     it('should use a caller-supplied route without performing discovery', () => {
@@ -523,7 +525,7 @@ describe('OpenFeature Exposures Writer', () => {
       ['broken pipe', Object.assign(new Error('write EPIPE'), { code: 'EPIPE' })],
       ['timeout', Object.assign(new Error('request timed out'), { code: 'ETIMEDOUT' })],
     ]) {
-      it(`should retry ambiguous local ${name} through direct intake and switch future batches`, async () => {
+      it(`should not replay ambiguous local ${name} and should switch future batches to direct intake`, async () => {
         const localUrl = new URL('http://serverless-init:8126')
         const directUrl = new URL('https://event-platform-intake.datadoghq.com')
         request.onFirstCall().yieldsAsync(error, null, statusCode)
@@ -546,15 +548,21 @@ describe('OpenFeature Exposures Writer', () => {
         writer.flush()
         await clock.tickAsync(0)
 
-        sinon.assert.calledTwice(request)
+        sinon.assert.calledOnce(request)
         assert.strictEqual(request.firstCall.args[1].url, localUrl)
-        assert.strictEqual(request.secondCall.args[1].url, directUrl)
+        sinon.assert.calledWithExactly(
+          log.error,
+          'Failed to send events to %s%s: %s',
+          'http://serverless-init:8126/',
+          '/evp_proxy/v4/api/v2/exposures',
+          error.message
+        )
 
         writer.append(exposureEvent)
         writer.flush()
 
-        sinon.assert.calledThrice(request)
-        assert.strictEqual(request.thirdCall.args[1].url, directUrl)
+        sinon.assert.calledTwice(request)
+        assert.strictEqual(request.secondCall.args[1].url, directUrl)
       })
     }