bench(sirun): make benchmark workloads deterministic

benchmark/sirun/appsec-iast/meta.json

@@ -15,7 +15,7 @@
         "DD_INJECT_FORCE": "true",
         "DD_IAST_ENABLED": "1",
         "DD_REMOTE_CONFIGURATION_ENABLED": "false",
-        "OPERATIONS": "30000"
+        "OPERATIONS": "28500"
       }
     },
     "with-vulnerability-iast-enabled-default-config": {
@@ -29,7 +29,7 @@
         "DD_INJECT_FORCE": "true",
         "DD_IAST_ENABLED": "1",
         "DD_REMOTE_CONFIGURATION_ENABLED": "false",
-        "OPERATIONS": "11500"
+        "OPERATIONS": "10000"
       }
     },
     "startup-time-iast-enabled": {

benchmark/sirun/debugger/meta.json

@@ -20,7 +20,7 @@
       "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node app.js\"",
       "env": {
         "STARTUP_GUARD_MAX_SHARE": "0.15",
-        "OPERATIONS": "17000",
+        "OPERATIONS": "11300",
         "BREAKPOINT_FILE": "app.js",
         "BREAKPOINT_LINE": "152",
         "EXPECTED_CAPTURE_KIND": "none",
@@ -47,7 +47,7 @@
       "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node app.js\"",
       "env": {
         "STARTUP_GUARD_MAX_SHARE": "0.15",
-        "OPERATIONS": "12000",
+        "OPERATIONS": "8000",
         "DD_DYNAMIC_INSTRUMENTATION_CAPTURE_TIMEOUT_MS": "1000",
         "BREAKPOINT_FILE": "app.js",
         "BREAKPOINT_LINE": "152",

benchmark/sirun/dogstatsd/index.js

@@ -1,8 +1,8 @@
 'use strict'
 
-const assert = require('node:assert/strict')
-
 const guard = require('../startup-guard')
+// eslint-disable-next-line import/order -- The startup guard must run before every other require.
+const assert = require('node:assert/strict')
 
 const { BASELINE_OR_CANDIDATE, VARIANT } = process.env
 const OPERATIONS = Number(process.env.OPERATIONS)

benchmark/sirun/dogstatsd/meta.json

@@ -10,7 +10,10 @@
       "env": { "VARIANT": "no-tags", "OPERATIONS": "6000000" }
     },
     "with-tags": {
-      "env": { "VARIANT": "with-tags", "OPERATIONS": "8300000" }
+      "run": "node --predictable-gc-schedule index.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --predictable-gc-schedule index.js\"",
+      "iterations": 30,
+      "env": { "VARIANT": "with-tags", "OPERATIONS": "3300000" }
     },
     "many-tags": {
       "env": { "VARIANT": "many-tags", "OPERATIONS": "3000000" }

benchmark/sirun/llmobs-span-processor/index.js

@@ -1,7 +1,8 @@
 'use strict'
 
-const assert = require('node:assert/strict')
 const guard = require('../startup-guard')
+// eslint-disable-next-line import/order -- The startup guard must run before every other require.
+const assert = require('node:assert/strict')
 
 globalThis[Symbol.for('dd-trace')] ??= { beforeExitHandlers: new Set() }
 

benchmark/sirun/llmobs-span-processor/meta.json

@@ -7,6 +7,8 @@
   "instructions": true,
   "variants": {
     "llm-chat": {
+      "run": "node --predictable-gc-schedule index.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --predictable-gc-schedule index.js\"",
       "env": { "VARIANT": "llm-chat", "OPERATIONS": "2000000" }
     },
     "embedding": {
@@ -16,7 +18,10 @@
       "env": { "VARIANT": "retrieval", "OPERATIONS": "2000000" }
     },
     "agent": {
-      "env": { "VARIANT": "agent", "OPERATIONS": "950000" }
+      "run": "node --predictable-gc-schedule index.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --predictable-gc-schedule index.js\"",
+      "iterations": 18,
+      "env": { "VARIANT": "agent", "OPERATIONS": "1600000" }
     }
   }
 }

benchmark/sirun/plugin-aws-sdk/index.js

@@ -1,7 +1,8 @@
 'use strict'
 
-const assert = require('node:assert/strict')
 const guard = require('../startup-guard')
+// eslint-disable-next-line import/order -- The startup guard must run before every other require.
+const assert = require('node:assert/strict')
 
 const BaseAwsSdkPlugin = require('../../../packages/datadog-plugin-aws-sdk/src/base')
 const EventBridge = require('../../../packages/datadog-plugin-aws-sdk/src/services/eventbridge')

benchmark/sirun/plugin-aws-sdk/meta.json

@@ -7,15 +7,18 @@
   "instructions": true,
   "variants": {
     "extract-response-body": {
+      "run": "node --predictable-gc-schedule index.js",
+      "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node --predictable-gc-schedule index.js\"",
       "env": {
         "VARIANT": "extract-response-body",
-        "OPERATIONS": "8000000"
+        "OPERATIONS": "27500000"
       }
     },
     "eventbridge-inject-detail": {
+      "iterations": 5,
       "env": {
         "VARIANT": "eventbridge-inject-detail",
-        "OPERATIONS": "3000000"
+        "OPERATIONS": "6000000"
       }
     },
     "lambda-inject-no-context": {
@@ -25,9 +28,10 @@
       }
     },
     "lambda-inject-with-context": {
+      "iterations": 16,
       "env": {
         "VARIANT": "lambda-inject-with-context",
-        "OPERATIONS": "2000000"
+        "OPERATIONS": "1330000"
       }
     }
   }

benchmark/sirun/plugin-claude-agent-sdk/meta.json

@@ -3,35 +3,36 @@
   "run": "node index.js",
   "run_with_affinity": "bash -c \"taskset -c $CPU_AFFINITY node index.js\"",
   "cachegrind": false,
-  "iterations": 15,
+  "iterations": 5,
   "instructions": true,
   "variants": {
     "compact-stream-scan": {
       "env": {
         "SCENARIO": "compact",
         "VARIANT": "stream-scan",
-        "OPERATIONS": "25000"
+        "OPERATIONS": "8000000"
       }
     },
     "compact-hook-indexed": {
       "env": {
         "SCENARIO": "compact",
         "VARIANT": "hook-indexed",
-        "OPERATIONS": "25000"
+        "OPERATIONS": "5300000"
       }
     },
     "delayed-noisy-stream-scan": {
+      "iterations": 10,
       "env": {
         "SCENARIO": "delayed-noisy",
         "VARIANT": "stream-scan",
-        "OPERATIONS": "25000"
+        "OPERATIONS": "125000"
       }
     },
     "delayed-noisy-hook-indexed": {
       "env": {
         "SCENARIO": "delayed-noisy",
         "VARIANT": "hook-indexed",
-        "OPERATIONS": "25000"
+        "OPERATIONS": "700000"
       }
     }
   }

benchmark/sirun/plugin-elasticsearch/meta.json

@@ -7,7 +7,8 @@
   "instructions": true,
   "variants": {
     "search": {
-      "env": { "VARIANT": "search", "OPERATIONS": "2000000" }
+      "iterations": 10,
+      "env": { "VARIANT": "search", "OPERATIONS": "4000000" }
     },
     "bulk-index": {
       "env": { "VARIANT": "bulk-index", "OPERATIONS": "274000" }

benchmark/sirun/plugin-graphql-long/index.js

@@ -1,21 +1,63 @@
 'use strict'
 
-// Long-workload graphql bench. Runs OPERATIONS queries per process (default 1500)
-// so the fixed startup cost (tracer load plus graphql require) stays a small
-// fraction of the run.
+// Long-workload graphql bench. Runs a fixed number of sequential queries per
+// process so tracer and graphql startup stay a small fraction of the run.
 
-const assert = require('node:assert/strict')
 const guard = require('../startup-guard')
+// eslint-disable-next-line import/order -- The startup guard must run before every other require.
+const assert = require('node:assert/strict')
+
+const exporterPath = require.resolve('../../../packages/dd-trace/src/exporter')
+const preflightTraces = []
+let capturePreflight = true
+
+class BenchmarkExporter {
+  /**
+   * @param {import('../../../packages/dd-trace/src/config')} config
+   */
+  constructor (config) {
+    this._url = config.url
+  }
+
+  /**
+   * @param {Array<Record<string, unknown>>} spans
+   */
+  export (spans) {
+    if (capturePreflight) {
+      preflightTraces.push(spans)
+    }
+  }
+
+  /**
+   * @param {URL} url
+   */
+  setUrl (url) {
+    this._url = url
+  }
+}
+
+function getBenchmarkExporter () {
+  return BenchmarkExporter
+}
 
 if (process.env.WITH_TRACER) {
+  require.cache[exporterPath] = {
+    id: exporterPath,
+    filename: exporterPath,
+    loaded: true,
+    exports: getBenchmarkExporter,
+  }
+
   const tracer = require('../../..').init()
 
-  if (process.env.WITH_DEPTH) {
-    tracer.use('graphql', { depth: Number(process.env.WITH_DEPTH) })
-  } else if (process.env.WITH_DEPTH_AND_COLLAPSE) {
-    const [depth, collapse] = process.env.WITH_DEPTH_AND_COLLAPSE.split(',')
-    tracer.use('graphql', { depth: Number(depth), collapse: Number(collapse) > 0 })
+  const options = {}
+  if (process.env.DEPTH) {
+    options.depth = Number(process.env.DEPTH)
   }
+  if (process.env.COLLAPSE) {
+    options.collapse = process.env.COLLAPSE === '1'
+  }
+  tracer.use('graphql', options)
 }
 
 const graphql = require('../../../versions/graphql').get()
@@ -42,28 +84,63 @@ const source = `
 
 const variableValues = { who: 'world' }
 
-// Total queries per process, run sequentially. graphql here is CPU-bound
-// (in-memory resolvers, nothing to overlap) and Node runs JS on one thread, so
-// the sequential loop already saturates the core (~96% CPU, measured). Keeping
-// several in flight only adds promise scheduling, live memory and GC, which made
-// the run slower and noisier, never faster.
+// Queries run sequentially. The resolvers return settled promises so the plugin
+// follows its Promise path without making libuv scheduling the measured surface.
 const OPERATIONS = Number(process.env.OPERATIONS)
 
-let checked = false
-
-guard.loopStart()
 ;(async () => {
+  const result = await graphql.graphql({ schema, source, variableValues })
+  assert.ok(result.data && !result.errors, 'graphql query returned no data')
+
+  if (process.env.WITH_TRACER) {
+    validatePreflightTrace()
+    capturePreflight = false
+    preflightTraces.length = 0
+  }
+
+  guard.loopStart()
   for (let i = 0; i < OPERATIONS; i++) {
-    const result = await graphql.graphql({ schema, source, variableValues })
-    if (!checked) {
-      // Fail loudly if the schema/resolvers stop producing a result: a broken
-      // bench that silently returns errors would otherwise still "pass".
-      assert.ok(result.data && !result.errors, 'graphql query returned no data')
-      checked = true
-    }
+    await graphql.graphql({ schema, source, variableValues })
   }
-  // Node 26 runs this loop ~2x faster than Node 20, so the fixed tracer.init()
-  // plus graphql-load setup is a larger share of the run there. Sizing OPERATIONS to
-  // keep it under the default 7% would push the Node 20 run past ~110s, so allow 10%.
+  // Node 26 runs this loop ~2x faster than Node 20. Per-Node operation counts
+  // keep the setup share below 10% without making slower releases overlong.
   guard.done(0.1)
 })()
+
+function validatePreflightTrace () {
+  let executeFound = false
+  let resolverCount = 0
+  let collapsedPathFound = false
+  let indexedPathFound = false
+
+  for (const trace of preflightTraces) {
+    for (const span of trace) {
+      if (span.name === 'graphql.execute') {
+        executeFound = true
+      } else if (span.name === 'graphql.resolve') {
+        resolverCount++
+        const path = span.meta['graphql.field.path']
+        collapsedPathFound ||= path.includes('.*.')
+        indexedPathFound ||= /\.\d+\./.test(path)
+      }
+    }
+  }
+
+  assert.ok(executeFound, 'graphql.execute span was not exported')
+
+  if (process.env.DEPTH === '0') {
+    assert.equal(resolverCount, 0, 'depth-off exported resolver spans')
+  } else {
+    assert.ok(resolverCount > 0, 'depth-on exported no resolver spans')
+    assert.equal(
+      collapsedPathFound,
+      process.env.COLLAPSE === '1',
+      'resolver path collapse did not match the variant'
+    )
+    assert.equal(
+      indexedPathFound,
+      process.env.COLLAPSE === '0',
+      'resolver path indexes did not match the variant'
+    )
+  }
+}