Rework histogram benchmarks to include exponential + no-bucket. (#3986)

* Rework histogram benchmarks to include exponential + no-bucket.

* spotless fix.

* Fix test scenarios from review.

* Add new benchmark for startup costs of histograms, with lots of caveats.

* Fixes from review.

sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramAggregationParam.java

@@ -0,0 +1,34 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.sdk.metrics.internal.aggregator;
+
+import io.opentelemetry.sdk.metrics.exemplar.ExemplarReservoir;
+import java.util.Collections;
+
+/** The types of histogram aggregation to benchmark. */
+@SuppressWarnings("ImmutableEnumChecker")
+public enum HistogramAggregationParam {
+  EXPLICIT_DEFAULT_BUCKET(
+      new DoubleHistogramAggregator(
+          ExplicitBucketHistogramUtils.createBoundaryArray(
+              ExplicitBucketHistogramUtils.DEFAULT_HISTOGRAM_BUCKET_BOUNDARIES),
+          ExemplarReservoir::noSamples)),
+  EXPLICIT_SINGLE_BUCKET(
+      new DoubleHistogramAggregator(
+          ExplicitBucketHistogramUtils.createBoundaryArray(Collections.emptyList()),
+          ExemplarReservoir::noSamples)),
+  EXPONENTIAL(new DoubleExponentialHistogramAggregator(ExemplarReservoir::noSamples));
+
+  private final Aggregator<?> aggregator;
+
+  private HistogramAggregationParam(Aggregator<?> aggregator) {
+    this.aggregator = aggregator;
+  }
+
+  public Aggregator<?> getAggregator() {
+    return this.aggregator;
+  }
+}

sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramBenchmark.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.sdk.metrics.internal.aggregator;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.DoubleSupplier;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+/** Measures runtime cost of histogram aggregations. */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Measurement(iterations = 10, time = 1)
+@Warmup(iterations = 5, time = 1)
+@Fork(1)
+public class HistogramBenchmark {
+
+  @State(Scope.Thread)
+  public static class ThreadState {
+    @Param HistogramValueGenerator valueGen;
+    @Param HistogramAggregationParam aggregation;
+    private AggregatorHandle<?> aggregatorHandle;
+    private DoubleSupplier valueSupplier;
+
+    @Setup(Level.Trial)
+    public final void setup() {
+      aggregatorHandle = aggregation.getAggregator().createHandle();
+      valueSupplier = valueGen.supplier();
+    }
+
+    public void record() {
+      // Record a number of samples.
+      for (int i = 0; i < 2000; i++) {
+        this.aggregatorHandle.recordDouble(valueSupplier.getAsDouble());
+      }
+    }
+  }
+
+  @Benchmark
+  @Threads(value = 10)
+  public void aggregate_10Threads(ThreadState threadState) {
+    threadState.record();
+  }
+
+  @Benchmark
+  @Threads(value = 5)
+  public void aggregate_5Threads(ThreadState threadState) {
+    threadState.record();
+  }
+
+  @Benchmark
+  @Threads(value = 1)
+  public void aggregate_1Threads(ThreadState threadState) {
+    threadState.record();
+  }
+}

sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramScaleBenchmark.java

@@ -0,0 +1,63 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.sdk.metrics.internal.aggregator;
+
+import java.util.concurrent.TimeUnit;
+import java.util.function.DoubleSupplier;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+
+/**
+ * Attempts to measure the cost of re-scaling/building buckets.
+ *
+ * <p>This benchmark must be interpreted carefully. We're looking for startup costs of histograms
+ * and need to tease out the portion of recorded time from scaling buckets vs. general algorithmic
+ * performance.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@Measurement(iterations = 10, time = 1)
+@Warmup(iterations = 5, time = 1)
+@Fork(1)
+public class HistogramScaleBenchmark {
+  @State(Scope.Thread)
+  public static class ThreadState {
+    @Param HistogramValueGenerator valueGen;
+    @Param HistogramAggregationParam aggregation;
+    private AggregatorHandle<?> aggregatorHandle;
+    private DoubleSupplier valueSupplier;
+
+    @Setup(Level.Invocation)
+    public final void setup() {
+      aggregatorHandle = aggregation.getAggregator().createHandle();
+      valueSupplier = valueGen.supplier();
+    }
+
+    public void record() {
+      // Record a number of samples.
+      for (int i = 0; i < 20000; i++) {
+        this.aggregatorHandle.recordDouble(valueSupplier.getAsDouble());
+      }
+    }
+  }
+
+  @Benchmark
+  @Threads(value = 1)
+  public void scaleUp(HistogramBenchmark.ThreadState threadState) {
+    threadState.record();
+  }
+}

sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramValueGenerator.java

@@ -0,0 +1,92 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package io.opentelemetry.sdk.metrics.internal.aggregator;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.DoubleSupplier;
+
+/** Methods of generating values for histogram benchmarks. */
+@SuppressWarnings("ImmutableEnumChecker")
+public enum HistogramValueGenerator {
+  // Test scenario where we rotate around histogram buckets.
+  // This is a degenerate test case where we see every next measurement in a different
+  // bucket, mean to be the "optimal" explicit bucket histogram scenario.
+  FIXED_BUCKET_BOUNDARIES(explicitDefaultBucketPool()),
+  // Test scenario where we randomly get values between 0 and 2000.
+  // Note: for millisecond latency, this would mean we expect our calls to be randomly
+  // distributed between 0 and 2 seconds (not very likely).
+  // This is meant to test more "worst case scenarios" where Exponential histograms must
+  // expand scale factor due to highly distributed data.
+  UNIFORM_RANDOM_WITHIN_2K(randomPool(20000, 2000)),
+  // Test scenario where we're measuring latency with mean of 1 seconds, std deviation of a quarter
+  // second.  This is our "optimised" use case.
+  // Note: In practice we likely want to add several gaussian pools, as in real microsevices we
+  // tend to notice some optional processing show up as additive gaussian noise with higher
+  // mean/stddev.  However, this best represents a simple microservice.
+  GAUSSIAN_LATENCY(randomGaussianPool(20000, 1000, 250));
+
+  // A random seed we use to ensure tests are repeatable.
+  private static final int INITIAL_SEED = 513423236;
+  private final double[] pool;
+
+  private HistogramValueGenerator(double[] pool) {
+    this.pool = pool;
+  }
+
+  /** Returns a supplier of doubles values. */
+  public final DoubleSupplier supplier() {
+    return new PoolSupplier(this.pool);
+  }
+
+  // Return values from the pool, rotating around as necessary back to the beginning.
+  private static class PoolSupplier implements DoubleSupplier {
+    private final double[] pool;
+    private final AtomicInteger idx = new AtomicInteger(0);
+
+    public PoolSupplier(double[] pool) {
+      this.pool = pool;
+    }
+
+    @Override
+    public double getAsDouble() {
+      return pool[idx.incrementAndGet() % pool.length];
+    }
+  }
+  /** Constructs a pool using explicit bucket histogram boundaries. */
+  private static double[] explicitDefaultBucketPool() {
+    List<Double> fixedBoundaries = new ArrayList<Double>();
+    // Add minimal recording value.
+    fixedBoundaries.add(0.0);
+    // Add the bucket LE bucket boundaries (starts at 5).
+    fixedBoundaries.addAll(ExplicitBucketHistogramUtils.DEFAULT_HISTOGRAM_BUCKET_BOUNDARIES);
+    // Add Double max value as our other extreme.
+    fixedBoundaries.add(Double.MAX_VALUE);
+    return ExplicitBucketHistogramUtils.createBoundaryArray(fixedBoundaries);
+  }
+
+  /** Create a pool of random numbers within bound, and of size. */
+  private static double[] randomPool(int size, double bound) {
+    double[] pool = new double[size];
+    Random random = new Random(INITIAL_SEED);
+    for (int i = 0; i < size; i++) {
+      pool[i] = random.nextDouble() * bound;
+    }
+    return pool;
+  }
+
+  /** Create a pool approximating a gaussian distribution w/ given mean and standard deviation. */
+  private static double[] randomGaussianPool(int size, double mean, double deviation) {
+    double[] pool = new double[size];
+    Random random = new Random(INITIAL_SEED);
+    for (int i = 0; i < size; i++) {
+      pool[i] = random.nextGaussian() * deviation + mean;
+    }
+    return pool;
+  }
+}