diff --git a/sdk/metrics/histograms.png b/sdk/metrics/histograms.png new file mode 100644 index 00000000000..a8ca2de8615 Binary files /dev/null and b/sdk/metrics/histograms.png differ diff --git a/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/DoubleHistogramBenchmark.java b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/DoubleHistogramBenchmark.java deleted file mode 100644 index a4c0aab5ebc..00000000000 --- a/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/DoubleHistogramBenchmark.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright The OpenTelemetry Authors - * SPDX-License-Identifier: Apache-2.0 - */ - -package io.opentelemetry.sdk.metrics.internal.aggregator; - -import io.opentelemetry.sdk.metrics.common.InstrumentType; -import io.opentelemetry.sdk.metrics.common.InstrumentValueType; -import io.opentelemetry.sdk.metrics.exemplar.ExemplarFilter; -import io.opentelemetry.sdk.metrics.internal.descriptor.InstrumentDescriptor; -import io.opentelemetry.sdk.metrics.view.Aggregation; -import java.util.Arrays; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -@State(Scope.Benchmark) -public class DoubleHistogramBenchmark { - private static final Aggregator aggregator = - Aggregation.explicitBucketHistogram(Arrays.asList(10.0, 100.0, 1_000.0)) - .createAggregator( - InstrumentDescriptor.create( - "name", "description", "1", InstrumentType.HISTOGRAM, InstrumentValueType.DOUBLE), - ExemplarFilter.neverSample()); - private AggregatorHandle aggregatorHandle; - - @Setup(Level.Trial) - public final void setup() { - aggregatorHandle = aggregator.createHandle(); - } - - @Benchmark - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - @Threads(value = 10) - public void aggregate_10Threads() { - aggregatorHandle.recordDouble(100.0056); - } - - @Benchmark - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - @Threads(value = 5) - public void aggregate_5Threads() { - aggregatorHandle.recordDouble(100.0056); - } - - @Benchmark - @Fork(1) - @Warmup(iterations = 5, time = 1) - @Measurement(iterations = 10, time = 1) - @BenchmarkMode(Mode.AverageTime) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - @Threads(value = 1) - public void aggregate_1Threads() { - aggregatorHandle.recordDouble(100.0056); - } -} diff --git a/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramAggregationParam.java b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramAggregationParam.java new file mode 100644 index 00000000000..e76f94c4143 --- /dev/null +++ b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramAggregationParam.java @@ -0,0 +1,34 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.metrics.internal.aggregator; + +import io.opentelemetry.sdk.metrics.exemplar.ExemplarReservoir; +import java.util.Collections; + +/** The types of histogram aggregation to benchmark. */ +@SuppressWarnings("ImmutableEnumChecker") +public enum HistogramAggregationParam { + EXPLICIT_DEFAULT_BUCKET( + new DoubleHistogramAggregator( + ExplicitBucketHistogramUtils.createBoundaryArray( + ExplicitBucketHistogramUtils.DEFAULT_HISTOGRAM_BUCKET_BOUNDARIES), + ExemplarReservoir::noSamples)), + EXPLICIT_SINGLE_BUCKET( + new DoubleHistogramAggregator( + ExplicitBucketHistogramUtils.createBoundaryArray(Collections.emptyList()), + ExemplarReservoir::noSamples)), + EXPONENTIAL(new DoubleExponentialHistogramAggregator(ExemplarReservoir::noSamples)); + + private final Aggregator aggregator; + + private HistogramAggregationParam(Aggregator aggregator) { + this.aggregator = aggregator; + } + + public Aggregator getAggregator() { + return this.aggregator; + } +} diff --git a/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramBenchmark.java b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramBenchmark.java new file mode 100644 index 00000000000..5379779a17c --- /dev/null +++ b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramBenchmark.java @@ -0,0 +1,70 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.metrics.internal.aggregator; + +import java.util.concurrent.TimeUnit; +import java.util.function.DoubleSupplier; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** Measures runtime cost of histogram aggregations. */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Measurement(iterations = 10, time = 1) +@Warmup(iterations = 5, time = 1) +@Fork(1) +public class HistogramBenchmark { + + @State(Scope.Thread) + public static class ThreadState { + @Param HistogramValueGenerator valueGen; + @Param HistogramAggregationParam aggregation; + private AggregatorHandle aggregatorHandle; + private DoubleSupplier valueSupplier; + + @Setup(Level.Trial) + public final void setup() { + aggregatorHandle = aggregation.getAggregator().createHandle(); + valueSupplier = valueGen.supplier(); + } + + public void record() { + // Record a number of samples. + for (int i = 0; i < 2000; i++) { + this.aggregatorHandle.recordDouble(valueSupplier.getAsDouble()); + } + } + } + + @Benchmark + @Threads(value = 10) + public void aggregate_10Threads(ThreadState threadState) { + threadState.record(); + } + + @Benchmark + @Threads(value = 5) + public void aggregate_5Threads(ThreadState threadState) { + threadState.record(); + } + + @Benchmark + @Threads(value = 1) + public void aggregate_1Threads(ThreadState threadState) { + threadState.record(); + } +} diff --git a/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramScaleBenchmark.java b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramScaleBenchmark.java new file mode 100644 index 00000000000..89b21643633 --- /dev/null +++ b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramScaleBenchmark.java @@ -0,0 +1,63 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.metrics.internal.aggregator; + +import java.util.concurrent.TimeUnit; +import java.util.function.DoubleSupplier; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +/** + * Attempts to measure the cost of re-scaling/building buckets. + * + *

This benchmark must be interpreted carefully. We're looking for startup costs of histograms + * and need to tease out the portion of recorded time from scaling buckets vs. general algorithmic + * performance. + */ +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Measurement(iterations = 10, time = 1) +@Warmup(iterations = 5, time = 1) +@Fork(1) +public class HistogramScaleBenchmark { + @State(Scope.Thread) + public static class ThreadState { + @Param HistogramValueGenerator valueGen; + @Param HistogramAggregationParam aggregation; + private AggregatorHandle aggregatorHandle; + private DoubleSupplier valueSupplier; + + @Setup(Level.Invocation) + public final void setup() { + aggregatorHandle = aggregation.getAggregator().createHandle(); + valueSupplier = valueGen.supplier(); + } + + public void record() { + // Record a number of samples. + for (int i = 0; i < 20000; i++) { + this.aggregatorHandle.recordDouble(valueSupplier.getAsDouble()); + } + } + } + + @Benchmark + @Threads(value = 1) + public void scaleUp(HistogramBenchmark.ThreadState threadState) { + threadState.record(); + } +} diff --git a/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramValueGenerator.java b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramValueGenerator.java new file mode 100644 index 00000000000..cbe48550dc6 --- /dev/null +++ b/sdk/metrics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramValueGenerator.java @@ -0,0 +1,92 @@ +/* + * Copyright The OpenTelemetry Authors + * SPDX-License-Identifier: Apache-2.0 + */ + +package io.opentelemetry.sdk.metrics.internal.aggregator; + +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.DoubleSupplier; + +/** Methods of generating values for histogram benchmarks. */ +@SuppressWarnings("ImmutableEnumChecker") +public enum HistogramValueGenerator { + // Test scenario where we rotate around histogram buckets. + // This is a degenerate test case where we see every next measurement in a different + // bucket, mean to be the "optimal" explicit bucket histogram scenario. + FIXED_BUCKET_BOUNDARIES(explicitDefaultBucketPool()), + // Test scenario where we randomly get values between 0 and 2000. + // Note: for millisecond latency, this would mean we expect our calls to be randomly + // distributed between 0 and 2 seconds (not very likely). + // This is meant to test more "worst case scenarios" where Exponential histograms must + // expand scale factor due to highly distributed data. + UNIFORM_RANDOM_WITHIN_2K(randomPool(20000, 2000)), + // Test scenario where we're measuring latency with mean of 1 seconds, std deviation of a quarter + // second. This is our "optimised" use case. + // Note: In practice we likely want to add several gaussian pools, as in real microsevices we + // tend to notice some optional processing show up as additive gaussian noise with higher + // mean/stddev. However, this best represents a simple microservice. + GAUSSIAN_LATENCY(randomGaussianPool(20000, 1000, 250)); + + // A random seed we use to ensure tests are repeatable. + private static final int INITIAL_SEED = 513423236; + private final double[] pool; + + private HistogramValueGenerator(double[] pool) { + this.pool = pool; + } + + /** Returns a supplier of doubles values. */ + public final DoubleSupplier supplier() { + return new PoolSupplier(this.pool); + } + + // Return values from the pool, rotating around as necessary back to the beginning. + private static class PoolSupplier implements DoubleSupplier { + private final double[] pool; + private final AtomicInteger idx = new AtomicInteger(0); + + public PoolSupplier(double[] pool) { + this.pool = pool; + } + + @Override + public double getAsDouble() { + return pool[idx.incrementAndGet() % pool.length]; + } + } + /** Constructs a pool using explicit bucket histogram boundaries. */ + private static double[] explicitDefaultBucketPool() { + List fixedBoundaries = new ArrayList(); + // Add minimal recording value. + fixedBoundaries.add(0.0); + // Add the bucket LE bucket boundaries (starts at 5). + fixedBoundaries.addAll(ExplicitBucketHistogramUtils.DEFAULT_HISTOGRAM_BUCKET_BOUNDARIES); + // Add Double max value as our other extreme. + fixedBoundaries.add(Double.MAX_VALUE); + return ExplicitBucketHistogramUtils.createBoundaryArray(fixedBoundaries); + } + + /** Create a pool of random numbers within bound, and of size. */ + private static double[] randomPool(int size, double bound) { + double[] pool = new double[size]; + Random random = new Random(INITIAL_SEED); + for (int i = 0; i < size; i++) { + pool[i] = random.nextDouble() * bound; + } + return pool; + } + + /** Create a pool approximating a gaussian distribution w/ given mean and standard deviation. */ + private static double[] randomGaussianPool(int size, double mean, double deviation) { + double[] pool = new double[size]; + Random random = new Random(INITIAL_SEED); + for (int i = 0; i < size; i++) { + pool[i] = random.nextGaussian() * deviation + mean; + } + return pool; + } +}