Add basic jmh overhead test (#3672)

* Add basic jmh overhead test

* Move javaagent arg to gradle file

* Rename module to benchmark-overhead-jmh

* Add imports

* Add comment

* Add benchmark for noop api

benchmark-overhead-jmh/build.gradle.kts

@@ -0,0 +1,49 @@
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+import me.champeau.jmh.JMHTask
+import net.ltgt.gradle.errorprone.errorprone
+
+plugins {
+  id("otel.java-conventions")
+  id("otel.jmh-conventions")
+}
+
+dependencies {
+  jmhImplementation("org.springframework.boot:spring-boot-starter-web:2.5.2")
+  jmhImplementation(project(":testing-common")) {
+    exclude("ch.qos.logback")
+  }
+
+  // this only exists to make Intellij happy since it doesn't (currently at least) understand our
+  // inclusion of this artifact inside of :testing-common
+  jmhCompileOnly(project(path = ":testing:armeria-shaded-for-testing", configuration = "shadow"))
+}
+
+tasks {
+
+  // TODO(trask) without disabling errorprone, jmh task fails with
+  //  Task :testing-overhead-jmh:jmhCompileGeneratedClasses FAILED
+  //   error: plug-in not found: ErrorProne
+  withType<JavaCompile>().configureEach {
+    options.errorprone {
+      isEnabled.set(false)
+    }
+  }
+
+  named<JMHTask>("jmh") {
+    val shadowTask = project(":javaagent").tasks.named<ShadowJar>("shadowJar").get()
+    inputs.files(layout.files(shadowTask))
+
+    // note: without an exporter, toSpanData() won't even be called
+    // (which is good for benchmarking the instrumentation itself)
+    val args = listOf(
+      "-javaagent:${shadowTask.archiveFile.get()}",
+      "-Dotel.traces.exporter=none",
+      "-Dotel.metrics.exporter=none"
+    )
+    // see https://github.com/melix/jmh-gradle-plugin/issues/200
+    jvmArgsPrepend.add(args.joinToString(" "))
+
+    // TODO(trask) is this ok? if it's ok, move to otel.jmh-conventions?
+    outputs.upToDateWhen { false }
+  }
+}

benchmark-overhead-jmh/src/jmh/java/io/opentelemetry/javaagent/benchmark/servlet/ServletBenchmark.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.benchmark.servlet;
+
+import io.opentelemetry.javaagent.benchmark.servlet.app.HelloWorldApplication;
+import io.opentelemetry.testing.internal.armeria.client.WebClient;
+import java.util.concurrent.TimeUnit;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+
+@BenchmarkMode(Mode.SampleTime)
+@OutputTimeUnit(TimeUnit.MICROSECONDS)
+@State(Scope.Thread)
+public class ServletBenchmark {
+
+  static {
+    // using static initializer instead of @Setup since only want to initialize the app under test
+    // once regardless of @State and @Threads
+    HelloWorldApplication.main();
+  }
+
+  // using shaded armeria http client from testing-common artifact since it won't be instrumented
+  private WebClient client;
+
+  @Setup
+  public void setup() {
+    client = WebClient.builder().build();
+  }
+
+  @TearDown
+  public void tearDown() {
+    HelloWorldApplication.stop();
+  }
+
+  @Benchmark
+  public Object execute() {
+    return client.get("http://localhost:8080").aggregate().join();
+  }
+}

benchmark-overhead-jmh/src/jmh/java/io/opentelemetry/javaagent/benchmark/servlet/ServletWithAgentDisabledBenchmark.java

@@ -0,0 +1,11 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.benchmark.servlet;
+
+import org.openjdk.jmh.annotations.Fork;
+
+@Fork(jvmArgsAppend = "-Dotel.javaagent.enabled=false")
+public class ServletWithAgentDisabledBenchmark extends ServletBenchmark {}

benchmark-overhead-jmh/src/jmh/java/io/opentelemetry/javaagent/benchmark/servlet/ServletWithNoopApiBenchmark.java

@@ -0,0 +1,11 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.benchmark.servlet;
+
+import org.openjdk.jmh.annotations.Fork;
+
+@Fork(jvmArgsAppend = "-Dotel.javaagent.experimental.use-noop-api=true")
+public class ServletWithNoopApiBenchmark extends ServletBenchmark {}

benchmark-overhead-jmh/src/jmh/java/io/opentelemetry/javaagent/benchmark/servlet/ServletWithOnePercentSamplingBenchmark.java

@@ -0,0 +1,11 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.benchmark.servlet;
+
+import org.openjdk.jmh.annotations.Fork;
+
+@Fork(jvmArgsAppend = {"-Dotel.traces.sampler=traceidratio", "-Dotel.traces.sampler.arg=0.01"})
+public class ServletWithOnePercentSamplingBenchmark extends ServletBenchmark {}

benchmark-overhead-jmh/src/jmh/java/io/opentelemetry/javaagent/benchmark/servlet/app/HelloWorldApplication.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.benchmark.servlet.app;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.ConfigurableApplicationContext;
+
+@SpringBootApplication
+public class HelloWorldApplication {
+
+  private static volatile ConfigurableApplicationContext context;
+
+  public static void main(String... args) {
+
+    context = SpringApplication.run(HelloWorldApplication.class, args);
+  }
+
+  public static void stop() {
+    context.stop();
+  }
+}

benchmark-overhead-jmh/src/jmh/java/io/opentelemetry/javaagent/benchmark/servlet/app/HelloWorldController.java

@@ -0,0 +1,18 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.javaagent.benchmark.servlet.app;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class HelloWorldController {
+
+  @GetMapping("/")
+  public String root() {
+    return "Hello world!";
+  }
+}

buildSrc/build.gradle.kts

@@ -45,6 +45,8 @@ dependencies {
   implementation("org.gradle:test-retry-gradle-plugin:1.2.1")
   // When updating, also update dependencyManagement/dependencyManagement.gradle.kts
   implementation("net.bytebuddy:byte-buddy-gradle-plugin:1.11.2")
+  implementation("gradle.plugin.io.morethan.jmhreport:gradle-jmh-report:0.9.0")
+  implementation("me.champeau.jmh:jmh-gradle-plugin:0.6.5")
   implementation("net.ltgt.gradle:gradle-errorprone-plugin:2.0.1")
   implementation("net.ltgt.gradle:gradle-nullaway-plugin:1.1.0")
 

buildSrc/src/main/kotlin/otel.jmh-conventions.gradle.kts

@@ -0,0 +1,36 @@
+plugins {
+  id("me.champeau.jmh")
+  id("io.morethan.jmhreport")
+}
+
+dependencies {
+  jmh(platform(project(":dependencyManagement")))
+  jmh("org.openjdk.jmh:jmh-core")
+  jmh("org.openjdk.jmh:jmh-generator-bytecode")
+}
+
+// invoke jmh on a single benchmark class like so:
+//   ./gradlew -PjmhIncludeSingleClass=StatsTraceContextBenchmark clean :grpc-core:jmh
+jmh {
+  failOnError.set(true)
+  resultFormat.set("JSON")
+  // Otherwise an error will happen:
+  // Could not expand ZIP 'byte-buddy-agent-1.9.7.jar'.
+  includeTests.set(false)
+  profilers.add("gc")
+  val jmhIncludeSingleClass: String? by project
+  if (jmhIncludeSingleClass != null) {
+    includes.add(jmhIncludeSingleClass as String)
+  }
+}
+
+jmhReport {
+  jmhResultPath = file("${buildDir}/results/jmh/results.json").absolutePath
+  jmhReportOutput = file("${buildDir}/results/jmh").absolutePath
+}
+
+tasks {
+  named("jmh") {
+    finalizedBy(named("jmhReport"))
+  }
+}

dependencyManagement/build.gradle.kts

@@ -60,6 +60,11 @@ val DEPENDENCY_SETS = listOf(
     "1.11.2",
     listOf("byte-buddy", "byte-buddy-agent", "byte-buddy-gradle-plugin")
   ),
+  DependencySet(
+    "org.openjdk.jmh",
+    "1.32",
+    listOf("jmh-core", "jmh-generator-bytecode")
+  ),
   DependencySet(
     "org.mockito",
     "3.11.1",

settings.gradle.kts

@@ -328,3 +328,4 @@ include(":instrumentation:wicket-8.0:javaagent")
 // benchmark
 include(":benchmark")
 include(":benchmark-e2e")
+include(":benchmark-overhead-jmh")