JMXFetch support for GraalVM Native (#8569)

* Add JMXFetch.run to TracerActivation
* Replace org.datadog.jmxfetch.Status.generateJson to fix the GraalVM native build error
* Replace org.datadog.jmxfetch.reporter.JsonReporter.doSendServiceCheck to fix the GraalVM native build error
* Add jmxfetch config resources needed at runtime
* Remove JsonProcessingException stub that won't be needed for com.datadoghq:jmxfetch:0.49.7+
* Adjust GraalVM Native substitutions for JMXFetch
* GraalVM Native JMXFetch smoke test
* Bump jmxfetch to 0.49.7 for GraalVM Native to resolve GraalVM native build errors related to json and yaml parsing libraries
* Add jmxfetch integrations metricconfigs for native builds
* Add more extensive error reporting related to metricconfigs for debugging purposes

Author: ygree

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -184,13 +184,11 @@ public static void start(
 
     if (Platform.isNativeImageBuilder()) {
       // these default services are not used during native-image builds
-      jmxFetchEnabled = false;
       remoteConfigEnabled = false;
       telemetryEnabled = false;
-      // apply trace instrumentation, but skip starting other services
+      // apply trace instrumentation, but skip other products at native-image build time
       startDatadogAgent(initTelemetry, inst);
       StaticEventLogger.end("Agent.start");
-
       return;
     }
 

dd-java-agent/agent-jmxfetch/build.gradle

@@ -11,7 +11,7 @@ plugins {
 apply from: "$rootDir/gradle/java.gradle"
 
 dependencies {
-  api('com.datadoghq:jmxfetch:0.49.6') {
+  api('com.datadoghq:jmxfetch:0.49.7') {
     exclude group: 'org.slf4j', module: 'slf4j-api'
     exclude group: 'org.slf4j', module: 'slf4j-jdk14'
     exclude group: 'com.beust', module: 'jcommander'

dd-java-agent/agent-jmxfetch/src/main/java/com/fasterxml/jackson/core/JsonProcessingException.java

@@ -9,6 +9,7 @@
 import datadog.trace.api.StatsDClient;
 import datadog.trace.api.StatsDClientManager;
 import datadog.trace.api.flare.TracerFlare;
+import datadog.trace.api.telemetry.LogCollector;
 import de.thetaphi.forbiddenapis.SuppressForbidden;
 import java.io.IOException;
 import java.io.InputStream;
@@ -174,6 +175,7 @@ private static List<String> getInternalMetricFiles() {
         log.debug("metricconfigs not found. returning empty set");
         return Collections.emptyList();
       }
+      log.debug("reading found metricconfigs");
       Scanner scanner = new Scanner(metricConfigsStream);
       scanner.useDelimiter("\n");
       final List<String> result = new ArrayList<>();
@@ -183,8 +185,19 @@ private static List<String> getInternalMetricFiles() {
         integrationName.clear();
         integrationName.add(config.replace(".yaml", ""));
 
-        if (Config.get().isJmxFetchIntegrationEnabled(integrationName, false)) {
+        if (!Config.get().isJmxFetchIntegrationEnabled(integrationName, false)) {
+          log.debug(
+              "skipping metric config `{}` because integration {} is disabled",
+              config,
+              integrationName);
+        } else {
           final URL resource = JMXFetch.class.getResource("metricconfigs/" + config);
+          if (resource == null) {
+            log.debug(
+                LogCollector.SEND_TELEMETRY, "metric config `{}` not found. skipping", config);
+            continue;
+          }
+          log.debug("adding metric config `{}`", config);
 
           // jar!/ means a file internal to a jar, only add the part after if it exists
           final String path = resource.getPath();

dd-java-agent/agent-jmxfetch/src/main/java/datadog/trace/agent/jmxfetch/JMXFetch.java

@@ -41,6 +41,7 @@ dependencies {
   api(project(':dd-java-agent:agent-bootstrap')) {
     exclude group: 'com.datadoghq', module: 'agent-logging'
   }
+  compileOnly project(':dd-java-agent:agent-jmxfetch')
   compileOnly project(':dd-java-agent:agent-profiling')
   api group: 'com.blogspot.mydailyjava', name: 'weak-lock-free', version: '0.17'
   api libs.bytebuddy

dd-java-agent/agent-tooling/build.gradle

@@ -2,8 +2,11 @@
 
 import com.datadog.profiling.controller.openjdk.JFREventContextIntegration;
 import datadog.communication.ddagent.SharedCommunicationObjects;
+import datadog.communication.monitor.DDAgentStatsDClientManager;
+import datadog.trace.agent.jmxfetch.JMXFetch;
 import datadog.trace.agent.tooling.ProfilerInstaller;
 import datadog.trace.agent.tooling.TracerInstaller;
+import datadog.trace.api.StatsDClientManager;
 import datadog.trace.bootstrap.instrumentation.api.ProfilingContextIntegration;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -20,6 +23,9 @@ public static void activate() {
           withProfiler
               ? new JFREventContextIntegration()
               : ProfilingContextIntegration.NoOp.INSTANCE);
+
+      StatsDClientManager statsDClientManager = DDAgentStatsDClientManager.statsDClientManager();
+      JMXFetch.run(statsDClientManager);
     } catch (Throwable e) {
       log.warn("Problem activating datadog tracer", e);
     }

dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/nativeimage/TracerActivation.java

@@ -37,7 +37,10 @@ public void methodAdvice(MethodTransformer transformer) {
   public String[] helperClassNames() {
     return new String[] {
       packageName + ".Target_datadog_jctools_counters_FixedSizeStripedLongCounterFields",
-      packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess"
+      packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess",
+      packageName + ".Target_org_datadog_jmxfetch_App",
+      packageName + ".Target_org_datadog_jmxfetch_Status",
+      packageName + ".Target_org_datadog_jmxfetch_reporter_JsonReporter",
     };
   }
 
@@ -49,7 +52,10 @@ public String[] muzzleIgnoredClassNames() {
       "jdk.vm.ci.meta.ResolvedJavaField",
       // ignore helper class names as usual
       packageName + ".Target_datadog_jctools_counters_FixedSizeStripedLongCounterFields",
-      packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess"
+      packageName + ".Target_datadog_jctools_util_UnsafeRefArrayAccess",
+      packageName + ".Target_org_datadog_jmxfetch_App",
+      packageName + ".Target_org_datadog_jmxfetch_Status",
+      packageName + ".Target_org_datadog_jmxfetch_reporter_JsonReporter",
     };
   }
 
@@ -58,6 +64,9 @@ public static class FindTargetClassesAdvice {
     public static void onExit(@Advice.Return(readOnly = false) List<Class<?>> result) {
       result.add(Target_datadog_jctools_counters_FixedSizeStripedLongCounterFields.class);
       result.add(Target_datadog_jctools_util_UnsafeRefArrayAccess.class);
+      result.add(Target_org_datadog_jmxfetch_App.class);
+      result.add(Target_org_datadog_jmxfetch_Status.class);
+      result.add(Target_org_datadog_jmxfetch_reporter_JsonReporter.class);
     }
   }
 }

dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/AnnotationSubstitutionProcessorInstrumentation.java

@@ -7,7 +7,11 @@
 import com.oracle.svm.core.jdk.Resources;
 import datadog.trace.agent.tooling.Instrumenter;
 import datadog.trace.agent.tooling.InstrumenterModule;
+import java.io.BufferedReader;
 import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
 import net.bytebuddy.asm.Advice;
 
 @AutoService(InstrumenterModule.class)
@@ -33,19 +37,45 @@ public static void onExit() {
       // (drop trace/shared prefixes from embedded resources, so we can find them in native-image
       // as the final executable won't have our isolating class-loader to map these resources)
 
-      String[] tracerResources = {
-        "dd-java-agent.version",
-        "dd-trace-api.version",
-        "trace/dd-trace-core.version",
-        "shared/dogstatsd/version.properties",
-        "shared/version-utils.version",
-        "shared/datadog/okhttp3/internal/publicsuffix/publicsuffixes.gz",
-        "profiling/jfr/dd.jfp",
-        "profiling/jfr/safepoints.jfp",
-        "profiling/jfr/overrides/comprehensive.jfp",
-        "profiling/jfr/overrides/minimal.jfp"
-      };
+      List<String> tracerResources = new ArrayList<>();
+      tracerResources.add("dd-java-agent.version");
+      tracerResources.add("dd-trace-api.version");
+      tracerResources.add("trace/dd-trace-core.version");
+      tracerResources.add("shared/dogstatsd/version.properties");
+      tracerResources.add("shared/version-utils.version");
+      tracerResources.add("shared/datadog/okhttp3/internal/publicsuffix/publicsuffixes.gz");
+      tracerResources.add("profiling/jfr/dd.jfp");
+      tracerResources.add("profiling/jfr/safepoints.jfp");
+      tracerResources.add("profiling/jfr/overrides/comprehensive.jfp");
+      tracerResources.add("profiling/jfr/overrides/minimal.jfp");
 
+      // jmxfetch configs
+      tracerResources.add(
+          "metrics/project.properties"); // org.datadog.jmxfetch.AppConfig reads its version
+      tracerResources.add("metrics/org/datadog/jmxfetch/default-jmx-metrics.yaml");
+      tracerResources.add("metrics/org/datadog/jmxfetch/new-gc-default-jmx-metrics.yaml");
+      tracerResources.add("metrics/org/datadog/jmxfetch/old-gc-default-jmx-metrics.yaml");
+
+      // tracer's jmxfetch configs
+      tracerResources.add("metrics/jmxfetch-config.yaml");
+      tracerResources.add("metrics/jmxfetch-websphere-config.yaml");
+
+      // jmxfetch integrations metricconfigs
+      String metricConfigsPath = "metrics/datadog/trace/agent/jmxfetch/";
+      String metricConfigs = metricConfigsPath + "metricconfigs.txt";
+      tracerResources.add(metricConfigs);
+      try (InputStream is = ClassLoader.getSystemResourceAsStream(metricConfigs);
+          BufferedReader reader = new BufferedReader(new InputStreamReader(is))) {
+        String metricConfig;
+        while ((metricConfig = reader.readLine()) != null) {
+          if (!metricConfig.trim().isEmpty()) {
+            tracerResources.add(metricConfigsPath + "metricconfigs/" + metricConfig);
+          }
+        }
+      } catch (Throwable ignore) {
+      }
+
+      // registering tracer resources to include in the native build
       for (String original : tracerResources) {
         String flattened = original.substring(original.indexOf('/') + 1);
         try (InputStream is = ClassLoader.getSystemResourceAsStream(original)) {

dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/ResourcesFeatureInstrumentation.java

@@ -0,0 +1,17 @@
+package datadog.trace.instrumentation.graal.nativeimage;
+
+import com.oracle.svm.core.annotate.Substitute;
+import com.oracle.svm.core.annotate.TargetClass;
+
+@TargetClass(className = "org.datadog.jmxfetch.App")
+public final class Target_org_datadog_jmxfetch_App {
+  @Substitute
+  private boolean getJsonConfigs() {
+    // This method has a reference to the excluded transitive dependency jackson-jr-objects.
+    // GraalVM Native detects it during the reachability analysis and results in
+    // "Discovered unresolved method during parsing:
+    // org.datadog.jmxfetch.App.<init>(org.datadog.jmxfetch.AppConfig)."
+    // because of the missing classes that belong to the excluded dependencies.
+    throw new IllegalStateException("Unreachable");
+  }
+}

dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_App.java

@@ -0,0 +1,17 @@
+package datadog.trace.instrumentation.graal.nativeimage;
+
+import com.oracle.svm.core.annotate.Substitute;
+import com.oracle.svm.core.annotate.TargetClass;
+import java.io.IOException;
+
+@TargetClass(className = "org.datadog.jmxfetch.Status")
+public final class Target_org_datadog_jmxfetch_Status {
+  @Substitute
+  private String generateJson() throws IOException {
+    // This method has a reference to the excluded transitive dependency jackson-jr-objects.
+    // GraalVM Native detects it during the reachability analysis and results in
+    // "Discovered unresolved type during parsing: com.fasterxml.jackson.jr.ob.JSON."
+    // because of the missing classes that belong to the excluded dependencies.
+    throw new IllegalStateException("Unreachable");
+  }
+}

dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_Status.java

@@ -0,0 +1,17 @@
+package datadog.trace.instrumentation.graal.nativeimage;
+
+import com.oracle.svm.core.annotate.Substitute;
+import com.oracle.svm.core.annotate.TargetClass;
+
+@TargetClass(className = "org.datadog.jmxfetch.reporter.JsonReporter")
+public final class Target_org_datadog_jmxfetch_reporter_JsonReporter {
+  @Substitute
+  public void doSendServiceCheck(
+      String serviceCheckName, String status, String message, String[] tags) {
+    // This method has a reference to the excluded transitive dependency jackson-jr-objects.
+    // GraalVM Native detects it during the reachability analysis and results in
+    // "Discovered unresolved type during parsing: com.fasterxml.jackson.jr.ob.JSON."
+    // because of the missing classes that belong to the excluded dependencies.
+    throw new IllegalStateException("Unreachable");
+  }
+}

dd-java-agent/instrumentation/graal/native-image/src/main/java/datadog/trace/instrumentation/graal/nativeimage/Target_org_datadog_jmxfetch_reporter_JsonReporter.java

@@ -39,6 +39,7 @@ if (hasProperty('agentPath')) {
         if (withProfiler && property('profiler') == 'true') {
           buildArgs.add("-J-Ddd.profiling.enabled=true")
         }
+        buildArgs.add("--enable-monitoring=jmxserver")
       }
     }
   }

dd-smoke-tests/spring-boot-3.0-native/application/build.gradle

@@ -1,4 +1,5 @@
 import datadog.smoketest.AbstractServerSmokeTest
+import datadog.trace.agent.test.utils.PortUtils
 import okhttp3.Request
 import org.openjdk.jmc.common.item.IItemCollection
 import org.openjdk.jmc.common.item.ItemFilters
@@ -13,6 +14,10 @@ import java.nio.file.Files
 import java.nio.file.Path
 import java.nio.file.SimpleFileVisitor
 import java.nio.file.attribute.BasicFileAttributes
+import java.util.concurrent.CompletableFuture
+import java.util.concurrent.Executors
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeoutException
 import java.util.concurrent.atomic.AtomicInteger
 import java.util.concurrent.locks.LockSupport
 
@@ -21,6 +26,9 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest {
   @TempDir
   def testJfrDir
 
+  @Shared
+  def statsdPort = PortUtils.randomOpenPort()
+
   @Override
   ProcessBuilder createProcessBuilder() {
     String springNativeExecutable = System.getProperty('datadog.smoketest.spring.native.executable')
@@ -39,7 +47,10 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest {
       '-Ddd.profiling.upload.period=1',
       '-Ddd.profiling.start-force-first=true',
       "-Ddd.profiling.debug.dump_path=${testJfrDir}",
-      "-Ddd.integration.spring-boot.enabled=true"
+      "-Ddd.integration.spring-boot.enabled=true",
+      "-Ddd.trace.debug=true",
+      "-Ddd.jmxfetch.statsd.port=${statsdPort}",
+      "-Ddd.jmxfetch.start-delay=0",
     ])
     ProcessBuilder processBuilder = new ProcessBuilder(command)
     processBuilder.directory(new File(buildDirectory))
@@ -66,8 +77,18 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest {
     super.isErrorLog(log) || log.contains("ClassNotFoundException")
   }
 
+  def setupSpec() {
+    try {
+      processTestLogLines { it.contains("JMXFetch config: ") }
+    } catch (TimeoutException toe) {
+      throw new AssertionError("'JMXFetch config: ' not found in logs. Make sure it's enabled.", toe)
+    }
+  }
+
   def "check native instrumentation"() {
     setup:
+    CompletableFuture<String> udpMessage = receiveUdpMessage(statsdPort, 1000)
+
     String url = "http://localhost:${httpPort}/hello"
 
     when:
@@ -87,6 +108,8 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest {
       LockSupport.parkNanos(1_000_000)
     }
     countJfrs() > 0
+
+    udpMessage.get(1, TimeUnit.SECONDS) contains "service:smoke-test-java-app,version:99,env:smoketest"
   }
 
   int countJfrs() {
@@ -115,4 +138,20 @@ class SpringBootNativeInstrumentationTest extends AbstractServerSmokeTest {
       })
     return jfrCount.get()
   }
+
+  CompletableFuture<String> receiveUdpMessage(int port, int bufferSize) {
+    def future = new CompletableFuture<String>()
+    Executors.newSingleThreadExecutor().submit {
+      try (DatagramSocket socket = new DatagramSocket(port)) {
+        byte[] buffer = new byte[bufferSize]
+        DatagramPacket packet = new DatagramPacket(buffer, buffer.length)
+        socket.receive(packet)
+        def received = new String(packet.data, 0, packet.length)
+        future.complete(received)
+      } catch (Exception e) {
+        future.completeExceptionally(e)
+      }
+    }
+    return future
+  }
 }