-
Notifications
You must be signed in to change notification settings - Fork 829
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Rework histogram benchmarks to include exponential + no-bucket. (#3986)
* Rework histogram benchmarks to include exponential + no-bucket. * spotless fix. * Fix test scenarios from review. * Add new benchmark for startup costs of histograms, with lots of caveats. * Fixes from review.
- Loading branch information
Showing
6 changed files
with
259 additions
and
75 deletions.
There are no files selected for viewing
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
75 changes: 0 additions & 75 deletions
75
...c/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/DoubleHistogramBenchmark.java
This file was deleted.
Oops, something went wrong.
34 changes: 34 additions & 0 deletions
34
.../jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramAggregationParam.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
/* | ||
* Copyright The OpenTelemetry Authors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package io.opentelemetry.sdk.metrics.internal.aggregator; | ||
|
||
import io.opentelemetry.sdk.metrics.exemplar.ExemplarReservoir; | ||
import java.util.Collections; | ||
|
||
/** The types of histogram aggregation to benchmark. */ | ||
@SuppressWarnings("ImmutableEnumChecker") | ||
public enum HistogramAggregationParam { | ||
EXPLICIT_DEFAULT_BUCKET( | ||
new DoubleHistogramAggregator( | ||
ExplicitBucketHistogramUtils.createBoundaryArray( | ||
ExplicitBucketHistogramUtils.DEFAULT_HISTOGRAM_BUCKET_BOUNDARIES), | ||
ExemplarReservoir::noSamples)), | ||
EXPLICIT_SINGLE_BUCKET( | ||
new DoubleHistogramAggregator( | ||
ExplicitBucketHistogramUtils.createBoundaryArray(Collections.emptyList()), | ||
ExemplarReservoir::noSamples)), | ||
EXPONENTIAL(new DoubleExponentialHistogramAggregator(ExemplarReservoir::noSamples)); | ||
|
||
private final Aggregator<?> aggregator; | ||
|
||
private HistogramAggregationParam(Aggregator<?> aggregator) { | ||
this.aggregator = aggregator; | ||
} | ||
|
||
public Aggregator<?> getAggregator() { | ||
return this.aggregator; | ||
} | ||
} |
70 changes: 70 additions & 0 deletions
70
...ics/src/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramBenchmark.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
/* | ||
* Copyright The OpenTelemetry Authors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package io.opentelemetry.sdk.metrics.internal.aggregator; | ||
|
||
import java.util.concurrent.TimeUnit; | ||
import java.util.function.DoubleSupplier; | ||
import org.openjdk.jmh.annotations.Benchmark; | ||
import org.openjdk.jmh.annotations.BenchmarkMode; | ||
import org.openjdk.jmh.annotations.Fork; | ||
import org.openjdk.jmh.annotations.Level; | ||
import org.openjdk.jmh.annotations.Measurement; | ||
import org.openjdk.jmh.annotations.Mode; | ||
import org.openjdk.jmh.annotations.OutputTimeUnit; | ||
import org.openjdk.jmh.annotations.Param; | ||
import org.openjdk.jmh.annotations.Scope; | ||
import org.openjdk.jmh.annotations.Setup; | ||
import org.openjdk.jmh.annotations.State; | ||
import org.openjdk.jmh.annotations.Threads; | ||
import org.openjdk.jmh.annotations.Warmup; | ||
|
||
/** Measures runtime cost of histogram aggregations. */ | ||
@BenchmarkMode(Mode.AverageTime) | ||
@OutputTimeUnit(TimeUnit.NANOSECONDS) | ||
@Measurement(iterations = 10, time = 1) | ||
@Warmup(iterations = 5, time = 1) | ||
@Fork(1) | ||
public class HistogramBenchmark { | ||
|
||
@State(Scope.Thread) | ||
public static class ThreadState { | ||
@Param HistogramValueGenerator valueGen; | ||
@Param HistogramAggregationParam aggregation; | ||
private AggregatorHandle<?> aggregatorHandle; | ||
private DoubleSupplier valueSupplier; | ||
|
||
@Setup(Level.Trial) | ||
public final void setup() { | ||
aggregatorHandle = aggregation.getAggregator().createHandle(); | ||
valueSupplier = valueGen.supplier(); | ||
} | ||
|
||
public void record() { | ||
// Record a number of samples. | ||
for (int i = 0; i < 2000; i++) { | ||
this.aggregatorHandle.recordDouble(valueSupplier.getAsDouble()); | ||
} | ||
} | ||
} | ||
|
||
@Benchmark | ||
@Threads(value = 10) | ||
public void aggregate_10Threads(ThreadState threadState) { | ||
threadState.record(); | ||
} | ||
|
||
@Benchmark | ||
@Threads(value = 5) | ||
public void aggregate_5Threads(ThreadState threadState) { | ||
threadState.record(); | ||
} | ||
|
||
@Benchmark | ||
@Threads(value = 1) | ||
public void aggregate_1Threads(ThreadState threadState) { | ||
threadState.record(); | ||
} | ||
} |
63 changes: 63 additions & 0 deletions
63
...rc/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramScaleBenchmark.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,63 @@ | ||
/* | ||
* Copyright The OpenTelemetry Authors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package io.opentelemetry.sdk.metrics.internal.aggregator; | ||
|
||
import java.util.concurrent.TimeUnit; | ||
import java.util.function.DoubleSupplier; | ||
import org.openjdk.jmh.annotations.Benchmark; | ||
import org.openjdk.jmh.annotations.BenchmarkMode; | ||
import org.openjdk.jmh.annotations.Fork; | ||
import org.openjdk.jmh.annotations.Level; | ||
import org.openjdk.jmh.annotations.Measurement; | ||
import org.openjdk.jmh.annotations.Mode; | ||
import org.openjdk.jmh.annotations.OutputTimeUnit; | ||
import org.openjdk.jmh.annotations.Param; | ||
import org.openjdk.jmh.annotations.Scope; | ||
import org.openjdk.jmh.annotations.Setup; | ||
import org.openjdk.jmh.annotations.State; | ||
import org.openjdk.jmh.annotations.Threads; | ||
import org.openjdk.jmh.annotations.Warmup; | ||
|
||
/** | ||
* Attempts to measure the cost of re-scaling/building buckets. | ||
* | ||
* <p>This benchmark must be interpreted carefully. We're looking for startup costs of histograms | ||
* and need to tease out the portion of recorded time from scaling buckets vs. general algorithmic | ||
* performance. | ||
*/ | ||
@BenchmarkMode(Mode.AverageTime) | ||
@OutputTimeUnit(TimeUnit.NANOSECONDS) | ||
@Measurement(iterations = 10, time = 1) | ||
@Warmup(iterations = 5, time = 1) | ||
@Fork(1) | ||
public class HistogramScaleBenchmark { | ||
@State(Scope.Thread) | ||
public static class ThreadState { | ||
@Param HistogramValueGenerator valueGen; | ||
@Param HistogramAggregationParam aggregation; | ||
private AggregatorHandle<?> aggregatorHandle; | ||
private DoubleSupplier valueSupplier; | ||
|
||
@Setup(Level.Invocation) | ||
public final void setup() { | ||
aggregatorHandle = aggregation.getAggregator().createHandle(); | ||
valueSupplier = valueGen.supplier(); | ||
} | ||
|
||
public void record() { | ||
// Record a number of samples. | ||
for (int i = 0; i < 20000; i++) { | ||
this.aggregatorHandle.recordDouble(valueSupplier.getAsDouble()); | ||
} | ||
} | ||
} | ||
|
||
@Benchmark | ||
@Threads(value = 1) | ||
public void scaleUp(HistogramBenchmark.ThreadState threadState) { | ||
threadState.record(); | ||
} | ||
} |
92 changes: 92 additions & 0 deletions
92
...rc/jmh/java/io/opentelemetry/sdk/metrics/internal/aggregator/HistogramValueGenerator.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
/* | ||
* Copyright The OpenTelemetry Authors | ||
* SPDX-License-Identifier: Apache-2.0 | ||
*/ | ||
|
||
package io.opentelemetry.sdk.metrics.internal.aggregator; | ||
|
||
import java.util.ArrayList; | ||
import java.util.List; | ||
import java.util.Random; | ||
import java.util.concurrent.atomic.AtomicInteger; | ||
import java.util.function.DoubleSupplier; | ||
|
||
/** Methods of generating values for histogram benchmarks. */ | ||
@SuppressWarnings("ImmutableEnumChecker") | ||
public enum HistogramValueGenerator { | ||
// Test scenario where we rotate around histogram buckets. | ||
// This is a degenerate test case where we see every next measurement in a different | ||
// bucket, mean to be the "optimal" explicit bucket histogram scenario. | ||
FIXED_BUCKET_BOUNDARIES(explicitDefaultBucketPool()), | ||
// Test scenario where we randomly get values between 0 and 2000. | ||
// Note: for millisecond latency, this would mean we expect our calls to be randomly | ||
// distributed between 0 and 2 seconds (not very likely). | ||
// This is meant to test more "worst case scenarios" where Exponential histograms must | ||
// expand scale factor due to highly distributed data. | ||
UNIFORM_RANDOM_WITHIN_2K(randomPool(20000, 2000)), | ||
// Test scenario where we're measuring latency with mean of 1 seconds, std deviation of a quarter | ||
// second. This is our "optimised" use case. | ||
// Note: In practice we likely want to add several gaussian pools, as in real microsevices we | ||
// tend to notice some optional processing show up as additive gaussian noise with higher | ||
// mean/stddev. However, this best represents a simple microservice. | ||
GAUSSIAN_LATENCY(randomGaussianPool(20000, 1000, 250)); | ||
|
||
// A random seed we use to ensure tests are repeatable. | ||
private static final int INITIAL_SEED = 513423236; | ||
private final double[] pool; | ||
|
||
private HistogramValueGenerator(double[] pool) { | ||
this.pool = pool; | ||
} | ||
|
||
/** Returns a supplier of doubles values. */ | ||
public final DoubleSupplier supplier() { | ||
return new PoolSupplier(this.pool); | ||
} | ||
|
||
// Return values from the pool, rotating around as necessary back to the beginning. | ||
private static class PoolSupplier implements DoubleSupplier { | ||
private final double[] pool; | ||
private final AtomicInteger idx = new AtomicInteger(0); | ||
|
||
public PoolSupplier(double[] pool) { | ||
this.pool = pool; | ||
} | ||
|
||
@Override | ||
public double getAsDouble() { | ||
return pool[idx.incrementAndGet() % pool.length]; | ||
} | ||
} | ||
/** Constructs a pool using explicit bucket histogram boundaries. */ | ||
private static double[] explicitDefaultBucketPool() { | ||
List<Double> fixedBoundaries = new ArrayList<Double>(); | ||
// Add minimal recording value. | ||
fixedBoundaries.add(0.0); | ||
// Add the bucket LE bucket boundaries (starts at 5). | ||
fixedBoundaries.addAll(ExplicitBucketHistogramUtils.DEFAULT_HISTOGRAM_BUCKET_BOUNDARIES); | ||
// Add Double max value as our other extreme. | ||
fixedBoundaries.add(Double.MAX_VALUE); | ||
return ExplicitBucketHistogramUtils.createBoundaryArray(fixedBoundaries); | ||
} | ||
|
||
/** Create a pool of random numbers within bound, and of size. */ | ||
private static double[] randomPool(int size, double bound) { | ||
double[] pool = new double[size]; | ||
Random random = new Random(INITIAL_SEED); | ||
for (int i = 0; i < size; i++) { | ||
pool[i] = random.nextDouble() * bound; | ||
} | ||
return pool; | ||
} | ||
|
||
/** Create a pool approximating a gaussian distribution w/ given mean and standard deviation. */ | ||
private static double[] randomGaussianPool(int size, double mean, double deviation) { | ||
double[] pool = new double[size]; | ||
Random random = new Random(INITIAL_SEED); | ||
for (int i = 0; i < size; i++) { | ||
pool[i] = random.nextGaussian() * deviation + mean; | ||
} | ||
return pool; | ||
} | ||
} |