Merge branch 'master' into alexeyk/fix-v1-span-events-json-encoding

# Conflicts:
#	dd-trace-core/src/test/java/datadog/trace/core/DDSpanSerializationTest.java

Author: AlexeyKuznetsov-DD

.gitlab-ci.yml

@@ -1115,7 +1115,7 @@ test_debugger_arm64:
     - !reference [.test_job_arm64, script]
 
 test_smoke:
-  extends: .test_job
+  extends: .test_job_with_test_agent
   # needs:parallel:matrix limits this job to a specific build_tests combination.
   # Keep matrix vars exact and in build_tests declaration order:
   # https://docs.gitlab.com/ci/yaml/#needsparallelmatrix
@@ -1135,7 +1135,7 @@ test_smoke:
     matrix: *test_matrix_8
 
 test_smoke_arm64:
-  extends: .test_job_arm64
+  extends: .test_job_arm64_with_test_agent
   variables:
     <<: *tier_l_variables
     GRADLE_TARGET: "stageMainDist :smokeTest"

communication/src/main/java/datadog/communication/ddagent/SharedCommunicationObjects.java

@@ -26,6 +26,8 @@
 public class SharedCommunicationObjects {
   private static final Logger log = LoggerFactory.getLogger(SharedCommunicationObjects.class);
 
+  private static final String X_DATADOG_TEST_SESSION_TOKEN = "X-Datadog-Test-Session-Token";
+
   private final List<Runnable> pausedComponents = new ArrayList<>();
   private volatile boolean paused;
 
@@ -90,9 +92,27 @@ public void createRemaining(Config config) {
       agentHttpClient =
           OkHttpUtils.buildHttpClient(
               OkHttpUtils.isPlainHttp(agentUrl), unixDomainSocket, namedPipe, httpClientTimeout);
+      String testSessionToken = config.getTestAgentSessionToken();
+      if (testSessionToken != null) {
+        agentHttpClient = injectTestAgentSessionHeaderInterceptor(testSessionToken);
+      }
     }
   }
 
+  private OkHttpClient injectTestAgentSessionHeaderInterceptor(String testSessionToken) {
+    return agentHttpClient
+        .newBuilder()
+        .addInterceptor(
+            chain ->
+                chain.proceed(
+                    chain
+                        .request()
+                        .newBuilder()
+                        .header(X_DATADOG_TEST_SESSION_TOKEN, testSessionToken)
+                        .build()))
+        .build();
+  }
+
   /** Registers a callback to be called when remote communications resume. */
   public void whenReady(Runnable callback) {
     if (paused) {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/instrumentation/java/concurrent/TPEHelper.java

@@ -6,6 +6,7 @@
 import datadog.context.ContextScope;
 import datadog.trace.api.GenericClassValue;
 import datadog.trace.api.InstrumenterConfig;
+import datadog.trace.api.Platform;
 import datadog.trace.bootstrap.ContextStore;
 import java.util.Set;
 import java.util.concurrent.ThreadPoolExecutor;
@@ -51,25 +52,15 @@ public static boolean useWrapping(Runnable task) {
     return useWrapping || task instanceof Wrapper || (task != null && WRAP.get(task.getClass()));
   }
 
-  public static void setPropagate(
-      ContextStore<ThreadPoolExecutor, Boolean> contextStore, ThreadPoolExecutor executor) {
-    if (executor == null || contextStore == null || contextStore.get(executor) != null) {
-      return;
-    }
-    String executorType = executor.getClass().getName();
-    if (excludedClasses.contains(executorType)) {
-      contextStore.put(executor, Boolean.FALSE);
-    } else {
-      contextStore.put(executor, Boolean.TRUE);
-    }
-  }
+  private static final ClassValue<Boolean> PROPAGATE =
+      GenericClassValue.of(input -> !excludedClasses.contains(input.getName()));
 
-  public static boolean shouldPropagate(
-      ContextStore<ThreadPoolExecutor, Boolean> contextStore, ThreadPoolExecutor executor) {
-    if (executor == null || contextStore == null) {
-      return false;
-    }
-    return Boolean.TRUE.equals(contextStore.get(executor));
+  public static boolean shouldPropagate(ThreadPoolExecutor executor) {
+    // avoid tracking threads when building native images as it confuses the scanner
+    // (we still want instrumentation applied, so tracking works in the built image)
+    return !Platform.isNativeImageBuilder()
+        && executor != null
+        && PROPAGATE.get(executor.getClass());
   }
 
   public static void capture(ContextStore<Runnable, State> contextStore, Runnable task) {

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ConfigurationFileLoader.java

@@ -1,30 +1,24 @@
 package com.datadog.debugger.agent;
 
-import com.datadog.debugger.probe.LogProbe;
-import com.datadog.debugger.probe.MetricProbe;
+import static com.datadog.debugger.probe.ProbeDefinitionDeserializer.deserializeLogProbe;
+import static com.datadog.debugger.probe.ProbeDefinitionDeserializer.deserializeMetricProbe;
+import static com.datadog.debugger.probe.ProbeDefinitionDeserializer.deserializeSpanDecorationProbe;
+import static com.datadog.debugger.probe.ProbeDefinitionDeserializer.deserializeSpanProbe;
+import static com.datadog.debugger.probe.ProbeDefinitionDeserializer.deserializeTriggerProbe;
+
 import com.datadog.debugger.probe.ProbeDefinition;
-import com.datadog.debugger.probe.SpanDecorationProbe;
-import com.datadog.debugger.probe.SpanProbe;
-import com.datadog.debugger.probe.TriggerProbe;
-import com.datadog.debugger.util.MoshiHelper;
 import com.squareup.moshi.JsonAdapter;
 import com.squareup.moshi.JsonReader;
 import com.squareup.moshi.JsonWriter;
-import com.squareup.moshi.Moshi;
-import com.squareup.moshi.Types;
 import datadog.trace.util.SizeCheckedInputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStream;
-import java.lang.annotation.Annotation;
-import java.lang.reflect.ParameterizedType;
-import java.lang.reflect.Type;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 import okio.Okio;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -46,9 +40,7 @@ public static Configuration from(Path probeFilePath, long maxPayloadSize) {
         }
       } while (bytesRead > -1);
       byte[] configContent = outputStream.toByteArray();
-      Moshi moshi = MoshiHelper.createMoshiConfigBuilder().add(new ProbeFileFactory()).build();
-      ParameterizedType type = Types.newParameterizedType(List.class, ProbeDefinition.class);
-      JsonAdapter<List<ProbeDefinition>> adapter = moshi.adapter(type);
+      JsonAdapter<List<ProbeDefinition>> adapter = new ProbeFileAdapter();
       List<ProbeDefinition> probeDefinitions =
           adapter.fromJson(
               JsonReader.of(Okio.buffer(Okio.source(new ByteArrayInputStream(configContent)))));
@@ -59,40 +51,7 @@ public static Configuration from(Path probeFilePath, long maxPayloadSize) {
     }
   }
 
-  private static class ProbeFileFactory implements JsonAdapter.Factory {
-    @Override
-    public JsonAdapter<?> create(Type type, Set<? extends Annotation> annotations, Moshi moshi) {
-      if (Types.equals(type, Types.newParameterizedType(List.class, ProbeDefinition.class))) {
-        return new ProbeFileAdapter(
-            moshi.adapter(LogProbe.class),
-            moshi.adapter(MetricProbe.class),
-            moshi.adapter(SpanProbe.class),
-            moshi.adapter(SpanDecorationProbe.class),
-            moshi.adapter(TriggerProbe.class));
-      }
-      return null;
-    }
-  }
-
   private static class ProbeFileAdapter extends JsonAdapter<List<ProbeDefinition>> {
-    private final JsonAdapter<LogProbe> logProbeAdapter;
-    private final JsonAdapter<MetricProbe> metricProbeAdapter;
-    private final JsonAdapter<SpanProbe> spanProbeAdapter;
-    private final JsonAdapter<SpanDecorationProbe> spanDecorationProbeAdapter;
-    private final JsonAdapter<TriggerProbe> triggerProbeAdapter;
-
-    public ProbeFileAdapter(
-        JsonAdapter<LogProbe> logProbeAdapter,
-        JsonAdapter<MetricProbe> metricProbeAdapter,
-        JsonAdapter<SpanProbe> spanProbeAdapter,
-        JsonAdapter<SpanDecorationProbe> spanDecorationProbeAdapter,
-        JsonAdapter<TriggerProbe> triggerProbeAdapter) {
-      this.logProbeAdapter = logProbeAdapter;
-      this.metricProbeAdapter = metricProbeAdapter;
-      this.spanProbeAdapter = spanProbeAdapter;
-      this.spanDecorationProbeAdapter = spanDecorationProbeAdapter;
-      this.triggerProbeAdapter = triggerProbeAdapter;
-    }
 
     @Override
     public List<ProbeDefinition> fromJson(JsonReader reader) throws IOException {
@@ -110,19 +69,19 @@ public List<ProbeDefinition> fromJson(JsonReader reader) throws IOException {
             String type = jsonPeekReader.nextString();
             switch (type) {
               case "LOG_PROBE":
-                probeDefinitions.add(logProbeAdapter.fromJson(reader));
+                probeDefinitions.add(deserializeLogProbe(reader));
                 break;
               case "METRIC_PROBE":
-                probeDefinitions.add(metricProbeAdapter.fromJson(reader));
+                probeDefinitions.add(deserializeMetricProbe(reader));
                 break;
               case "SPAN_PROBE":
-                probeDefinitions.add(spanProbeAdapter.fromJson(reader));
+                probeDefinitions.add(deserializeSpanProbe(reader));
                 break;
               case "SPAN_DECORATION_PROBE":
-                probeDefinitions.add(spanDecorationProbeAdapter.fromJson(reader));
+                probeDefinitions.add(deserializeSpanDecorationProbe(reader));
                 break;
               case "TRIGGER_PROBE":
-                probeDefinitions.add(triggerProbeAdapter.fromJson(reader));
+                probeDefinitions.add(deserializeTriggerProbe(reader));
                 break;
               default:
                 throw new RuntimeException("Unknown type: " + type);

dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/probe/ProbeDefinitionDeserializer.java

@@ -3,6 +3,7 @@
 import com.datadog.debugger.agent.Configuration;
 import com.datadog.debugger.util.MoshiHelper;
 import com.squareup.moshi.JsonAdapter;
+import com.squareup.moshi.JsonReader;
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import okio.Okio;
@@ -29,22 +30,42 @@ public static MetricProbe deserializeMetricProbe(byte[] content) throws IOExcept
     return deserialize(METRIC_PROBE_JSON_ADAPTER, content);
   }
 
+  public static MetricProbe deserializeMetricProbe(JsonReader reader) throws IOException {
+    return METRIC_PROBE_JSON_ADAPTER.fromJson(reader);
+  }
+
   public static LogProbe deserializeLogProbe(byte[] content) throws IOException {
     LogProbe logProbe = deserialize(LOG_PROBE_JSON_ADAPTER, content);
     logProbe.initSamplers();
     return logProbe;
   }
 
+  public static LogProbe deserializeLogProbe(JsonReader reader) throws IOException {
+    LogProbe logProbe = LOG_PROBE_JSON_ADAPTER.fromJson(reader);
+    logProbe.initSamplers();
+    return logProbe;
+  }
+
   public static SpanProbe deserializeSpanProbe(byte[] content) throws IOException {
     return deserialize(SPAN_PROBE_JSON_ADAPTER, content);
   }
 
+  public static SpanProbe deserializeSpanProbe(JsonReader reader) throws IOException {
+    return SPAN_PROBE_JSON_ADAPTER.fromJson(reader);
+  }
+
   public static TriggerProbe deserializeTriggerProbe(byte[] content) throws IOException {
     TriggerProbe triggerProbe = deserialize(TRIGGER_PROBE_JSON_ADAPTER, content);
     triggerProbe.initSamplers();
     return triggerProbe;
   }
 
+  public static TriggerProbe deserializeTriggerProbe(JsonReader jsonReader) throws IOException {
+    TriggerProbe triggerProbe = TRIGGER_PROBE_JSON_ADAPTER.fromJson(jsonReader);
+    triggerProbe.initSamplers();
+    return triggerProbe;
+  }
+
   public static SpanDecorationProbe deserializeSpanDecorationProbe(byte[] content)
       throws IOException {
     SpanDecorationProbe spanDecorationProbe =
@@ -53,6 +74,14 @@ public static SpanDecorationProbe deserializeSpanDecorationProbe(byte[] content)
     return spanDecorationProbe;
   }
 
+  public static SpanDecorationProbe deserializeSpanDecorationProbe(JsonReader jsonReader)
+      throws IOException {
+    SpanDecorationProbe spanDecorationProbe =
+        SPAN_DECORATION_PROBE_JSON_ADAPTER.fromJson(jsonReader);
+    spanDecorationProbe.initSamplers();
+    return spanDecorationProbe;
+  }
+
   private static <T> T deserialize(JsonAdapter<T> adapter, byte[] content) throws IOException {
     return adapter.fromJson(Okio.buffer(Okio.source(new ByteArrayInputStream(content))));
   }

dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/agent/ConfigurationFileLoaderTest.java

@@ -7,6 +7,7 @@
 import com.datadog.debugger.probe.ProbeDefinition;
 import com.datadog.debugger.probe.SpanDecorationProbe;
 import com.datadog.debugger.probe.SpanProbe;
+import com.datadog.debugger.probe.TriggerProbe;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
@@ -21,11 +22,12 @@ public void load() throws Exception {
     Configuration configuration = ConfigurationFileLoader.from(probeFilePath, 1024 * 1024);
     assertNotNull(configuration);
     List<ProbeDefinition> definitions = configuration.getDefinitions();
-    assertEquals(5, definitions.size());
-    assertInstanceOf(MetricProbe.class, definitions.get(0));
-    assertInstanceOf(LogProbe.class, definitions.get(1));
+    assertEquals(6, definitions.size());
+    assertInstanceOf(TriggerProbe.class, definitions.get(0));
+    assertInstanceOf(MetricProbe.class, definitions.get(1));
     assertInstanceOf(LogProbe.class, definitions.get(2));
-    assertInstanceOf(SpanProbe.class, definitions.get(3));
-    assertInstanceOf(SpanDecorationProbe.class, definitions.get(4));
+    assertInstanceOf(LogProbe.class, definitions.get(3));
+    assertInstanceOf(SpanProbe.class, definitions.get(4));
+    assertInstanceOf(SpanDecorationProbe.class, definitions.get(5));
   }
 }

dd-java-agent/agent-debugger/src/test/resources/test_probe_file.json

@@ -87,5 +87,19 @@
     },
     "tags": [],
     "evaluateAt": "EXIT"
+  },
+  {
+    "id": "70b55d06-f9fa-403b-a329-4f2f960aed02",
+    "version": 0,
+    "type": "TRIGGER_PROBE",
+    "language": "java",
+    "where": {
+      "typeName": "MetadataClientUtils",
+      "methodName": "listTableWithContinuation"
+    },
+    "tags": [],
+    "evaluateAt": "EXIT",
+    "sessionId": "session",
+    "sampling": { "snapshotsPerSecond": 100 }
   }
 ]

dd-java-agent/appsec/src/main/java/com/datadog/appsec/ddwaf/WAFModule.java

@@ -1,6 +1,5 @@
 package com.datadog.appsec.ddwaf;
 
-import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY;
 import static datadog.trace.util.stacktrace.StackTraceEvent.DEFAULT_LANGUAGE;
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
@@ -622,7 +621,10 @@ private Waf.ResultWithData runWafTransient(
   private Collection<AppSecEvent> buildEvents(
       Waf.ResultWithData actionWithData, String securityResponseId) {
     if (actionWithData.data == null) {
-      log.debug(SEND_TELEMETRY, "WAF result data is null");
+      // Since ruleset 1.14.1, fingerprint processors evaluate unconditionally, so the WAF
+      // returns MATCH with no data (no real rule/event) on every ordinary request, not just on
+      // attacks. This is expected and no longer worth logging (dd-trace-py/dd-trace-go don't log
+      // here either: both gate their equivalent log on non-empty events, not on the return code).
       return Collections.emptyList();
     }
     Collection<WAFResultData> listResults;