-
Notifications
You must be signed in to change notification settings - Fork 29
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Normalize metrics from measurements #45
base: master
Are you sure you want to change the base?
Changes from 1 commit
23cd790
a75646c
87c507a
0f2f138
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -13,6 +13,7 @@ | |
*/ | ||
package io.trino.benchto.driver.graphite; | ||
|
||
import com.google.common.collect.ImmutableMap; | ||
import io.trino.benchto.driver.Measurable; | ||
import io.trino.benchto.driver.execution.BenchmarkExecutionResult; | ||
import io.trino.benchto.driver.execution.ExecutionSynchronizer; | ||
|
@@ -32,6 +33,7 @@ | |
|
||
import java.time.ZonedDateTime; | ||
import java.time.temporal.ChronoUnit; | ||
import java.util.Collections; | ||
import java.util.List; | ||
import java.util.Map; | ||
import java.util.Optional; | ||
|
@@ -49,6 +51,7 @@ | |
public class GraphiteMetricsLoader | ||
implements PostExecutionMeasurementProvider | ||
{ | ||
private static final String METRIC_SCOPE = "cluster"; | ||
nineinchnick marked this conversation as resolved.
Show resolved
Hide resolved
|
||
private static final Logger LOG = LoggerFactory.getLogger(GraphiteMetricsLoader.class); | ||
|
||
@Autowired | ||
|
@@ -126,7 +129,7 @@ private List<Measurement> doLoadMeasurements(long fromEpochSecond, long toEpochS | |
if (metricValues.length > 0) { | ||
// last non zero measurement contains total over time | ||
double totalBytes = getLastValueGreaterThanZero(metricValues); | ||
measurements.add(Measurement.measurement("cluster-network_total", "BYTES", totalBytes)); | ||
measurements.add(Measurement.measurement("network", "BYTES", totalBytes, ImmutableMap.of("scope", METRIC_SCOPE, "aggregate", "total"))); | ||
} | ||
} | ||
return measurements; | ||
|
@@ -150,8 +153,8 @@ private void addMeanMaxMeasurements(Map<String, double[]> loadedMetrics, List<Me | |
{ | ||
Optional<StatisticalSummary> statistics = getStats(loadedMetrics, metricName); | ||
if (statistics.isPresent()) { | ||
measurements.add(Measurement.measurement("cluster-" + metricName + "_max", unit, statistics.get().getMax())); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. won't this make scripts harder, e.g: previously I could just select name in SQL query. Now I will have to join on attributes and do subquery. Seems complicated |
||
measurements.add(Measurement.measurement("cluster-" + metricName + "_mean", unit, statistics.get().getMean())); | ||
measurements.add(Measurement.measurement(metricName, unit, statistics.get().getMax(), ImmutableMap.of("scope", "cluster", "aggregate", "max"))); | ||
nineinchnick marked this conversation as resolved.
Show resolved
Hide resolved
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
measurements.add(Measurement.measurement(metricName, unit, statistics.get().getMean(), ImmutableMap.of("scope", "cluster", "aggregate", "mean"))); | ||
} | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -34,6 +34,7 @@ | |
import javax.measure.unit.Unit; | ||
|
||
import java.net.URI; | ||
import java.util.Collections; | ||
import java.util.List; | ||
import java.util.Map; | ||
|
||
|
@@ -123,7 +124,7 @@ private URI buildQueryInfoURI(String queryId) | |
private Measurement parseQueryStatistic(String name, Object statistic, Unit requiredUnit) | ||
{ | ||
double value = UnitConverter.parseValueAsUnit(statistic.toString(), requiredUnit); | ||
return measurement("prestoQuery-" + name, UnitConverter.format(requiredUnit), value); | ||
return measurement(name, UnitConverter.format(requiredUnit), value, Collections.singletonMap("scope", "prestoQuery")); | ||
nineinchnick marked this conversation as resolved.
Show resolved
Hide resolved
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
|
||
} | ||
|
||
@SuppressWarnings("unused") | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -21,7 +21,9 @@ | |
import org.springframework.beans.factory.annotation.Autowired; | ||
|
||
import java.io.IOException; | ||
import java.util.Collections; | ||
import java.util.List; | ||
import java.util.Map; | ||
|
||
import static org.assertj.core.api.Assertions.assertThat; | ||
import static org.springframework.http.MediaType.APPLICATION_JSON; | ||
|
@@ -44,20 +46,21 @@ public void testPrestoClientLoadMetrics() | |
|
||
List<Measurement> measurements = prestoClient.loadMetrics("test_query_id"); | ||
|
||
Map<String, String> attributes = Collections.singletonMap("scope", "prestoQuery"); | ||
assertThat(measurements).containsExactly( | ||
Measurement.measurement("prestoQuery-analysisTime", "MILLISECONDS", 21.07), | ||
Measurement.measurement("prestoQuery-planningTime", "MILLISECONDS", 24.72), | ||
Measurement.measurement("prestoQuery-totalScheduledTime", "MILLISECONDS", 66000.0), | ||
Measurement.measurement("prestoQuery-totalCpuTime", "MILLISECONDS", 63600.0), | ||
Measurement.measurement("prestoQuery-totalBlockedTime", "MILLISECONDS", 287400.0), | ||
Measurement.measurement("prestoQuery-finishingTime", "MILLISECONDS", 69000.0), | ||
Measurement.measurement("prestoQuery-rawInputDataSize", "BYTES", 1.34E9), | ||
Measurement.measurement("prestoQuery-processedInputDataSize", "BYTES", 7.3961E8), | ||
Measurement.measurement("prestoQuery-internalNetworkInputDataSize", "BYTES", 7.2961E8), | ||
Measurement.measurement("prestoQuery-physicalInputDataSize", "BYTES", 1.35E9), | ||
Measurement.measurement("prestoQuery-outputDataSize", "BYTES", 6900.0), | ||
Measurement.measurement("prestoQuery-peakTotalMemoryReservation", "BYTES", 6800.0), | ||
Measurement.measurement("prestoQuery-physicalWrittenDataSize", "BYTES", 462265065.0)); | ||
Measurement.measurement("analysisTime", "MILLISECONDS", 21.07, attributes), | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this will break scripts (jupyther). I think it's fine, but needs to be announced There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Not only scripts but our workflows as well. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You mean alerts? |
||
Measurement.measurement("planningTime", "MILLISECONDS", 24.72, attributes), | ||
Measurement.measurement("totalScheduledTime", "MILLISECONDS", 66000.0, attributes), | ||
Measurement.measurement("totalCpuTime", "MILLISECONDS", 63600.0, attributes), | ||
Measurement.measurement("totalBlockedTime", "MILLISECONDS", 287400.0, attributes), | ||
Measurement.measurement("finishingTime", "MILLISECONDS", 69000.0, attributes), | ||
Measurement.measurement("rawInputDataSize", "BYTES", 1.34E9, attributes), | ||
Measurement.measurement("processedInputDataSize", "BYTES", 7.3961E8, attributes), | ||
Measurement.measurement("internalNetworkInputDataSize", "BYTES", 7.2961E8, attributes), | ||
Measurement.measurement("physicalInputDataSize", "BYTES", 1.35E9, attributes), | ||
Measurement.measurement("outputDataSize", "BYTES", 6900.0, attributes), | ||
Measurement.measurement("peakTotalMemoryReservation", "BYTES", 6800.0, attributes), | ||
Measurement.measurement("physicalWrittenDataSize", "BYTES", 462265065.0, attributes)); | ||
|
||
restServiceServer.verify(); | ||
} | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
please add description in commit message