Skip to content

Commit

Permalink
Unconditionally include the spawn metrics in the execution log and de…
Browse files Browse the repository at this point in the history 
…lete the --experimental_execution_log_spawn_metrics flag.

PiperOrigin-RevId: 563015767
Change-Id: I49963f0b0ec6f9da3bbad90a6ea80abb467520a1
  • Loading branch information
@tjgq @copybara-github
tjgq authored and copybara-github committed Sep 6, 2023
1 parent 3c7c9db commit c0f2e14
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 64 deletions.
8 changes: 0 additions & 8 deletions src/main/java/com/google/devtools/build/lib/exec/ExecutionOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,14 +441,6 @@ public boolean usingLocalTestJobs() {
+ " --subcommands (for displaying subcommands in terminal output).")
public PathFragment executionLogBinaryFile;

@Option(
name = "experimental_execution_log_spawn_metrics",
defaultValue = "false",
documentationCategory = OptionDocumentationCategory.UNCATEGORIZED,
effectTags = {OptionEffectTag.UNKNOWN},
help = "Include spawn metrics in the executed spawns log.")
public boolean executionLogSpawnMetrics;

@Option(
name = "execution_log_json_file",
defaultValue = "null",
Expand Down
86 changes: 42 additions & 44 deletions src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,50 +177,48 @@ public void logSpawn(
builder.setTargetLabel(spawn.getTargetLabel());
}

if (executionOptions != null && executionOptions.executionLogSpawnMetrics) {
SpawnMetrics metrics = result.getMetrics();
Protos.SpawnMetrics.Builder metricsBuilder = builder.getMetricsBuilder();
if (metrics.totalTimeInMs() != 0L) {
metricsBuilder.setTotalTime(millisToProto(metrics.totalTimeInMs()));
}
if (metrics.parseTimeInMs() != 0L) {
metricsBuilder.setParseTime(millisToProto(metrics.parseTimeInMs()));
}
if (metrics.networkTimeInMs() != 0L) {
metricsBuilder.setNetworkTime(millisToProto(metrics.networkTimeInMs()));
}
if (metrics.fetchTimeInMs() != 0L) {
metricsBuilder.setFetchTime(millisToProto(metrics.fetchTimeInMs()));
}
if (metrics.queueTimeInMs() != 0L) {
metricsBuilder.setQueueTime(millisToProto(metrics.queueTimeInMs()));
}
if (metrics.setupTimeInMs() != 0L) {
metricsBuilder.setSetupTime(millisToProto(metrics.setupTimeInMs()));
}
if (metrics.uploadTimeInMs() != 0L) {
metricsBuilder.setUploadTime(millisToProto(metrics.uploadTimeInMs()));
}
if (metrics.executionWallTimeInMs() != 0L) {
metricsBuilder.setExecutionWallTime(millisToProto(metrics.executionWallTimeInMs()));
}
if (metrics.processOutputsTimeInMs() != 0L) {
metricsBuilder.setProcessOutputsTime(millisToProto(metrics.processOutputsTimeInMs()));
}
if (metrics.retryTimeInMs() != 0L) {
metricsBuilder.setRetryTime(millisToProto(metrics.retryTimeInMs()));
}
metricsBuilder.setInputBytes(metrics.inputBytes());
metricsBuilder.setInputFiles(metrics.inputFiles());
metricsBuilder.setMemoryEstimateBytes(metrics.memoryEstimate());
metricsBuilder.setInputBytesLimit(metrics.inputBytesLimit());
metricsBuilder.setInputFilesLimit(metrics.inputFilesLimit());
metricsBuilder.setOutputBytesLimit(metrics.outputBytesLimit());
metricsBuilder.setOutputFilesLimit(metrics.outputFilesLimit());
metricsBuilder.setMemoryBytesLimit(metrics.memoryLimit());
if (metrics.timeLimitInMs() != 0L) {
metricsBuilder.setTimeLimit(millisToProto(metrics.timeLimitInMs()));
}
SpawnMetrics metrics = result.getMetrics();
Protos.SpawnMetrics.Builder metricsBuilder = builder.getMetricsBuilder();
if (metrics.totalTimeInMs() != 0L) {
metricsBuilder.setTotalTime(millisToProto(metrics.totalTimeInMs()));
}
if (metrics.parseTimeInMs() != 0L) {
metricsBuilder.setParseTime(millisToProto(metrics.parseTimeInMs()));
}
if (metrics.networkTimeInMs() != 0L) {
metricsBuilder.setNetworkTime(millisToProto(metrics.networkTimeInMs()));
}
if (metrics.fetchTimeInMs() != 0L) {
metricsBuilder.setFetchTime(millisToProto(metrics.fetchTimeInMs()));
}
if (metrics.queueTimeInMs() != 0L) {
metricsBuilder.setQueueTime(millisToProto(metrics.queueTimeInMs()));
}
if (metrics.setupTimeInMs() != 0L) {
metricsBuilder.setSetupTime(millisToProto(metrics.setupTimeInMs()));
}
if (metrics.uploadTimeInMs() != 0L) {
metricsBuilder.setUploadTime(millisToProto(metrics.uploadTimeInMs()));
}
if (metrics.executionWallTimeInMs() != 0L) {
metricsBuilder.setExecutionWallTime(millisToProto(metrics.executionWallTimeInMs()));
}
if (metrics.processOutputsTimeInMs() != 0L) {
metricsBuilder.setProcessOutputsTime(millisToProto(metrics.processOutputsTimeInMs()));
}
if (metrics.retryTimeInMs() != 0L) {
metricsBuilder.setRetryTime(millisToProto(metrics.retryTimeInMs()));
}
metricsBuilder.setInputBytes(metrics.inputBytes());
metricsBuilder.setInputFiles(metrics.inputFiles());
metricsBuilder.setMemoryEstimateBytes(metrics.memoryEstimate());
metricsBuilder.setInputBytesLimit(metrics.inputBytesLimit());
metricsBuilder.setInputFilesLimit(metrics.inputFilesLimit());
metricsBuilder.setOutputBytesLimit(metrics.outputBytesLimit());
metricsBuilder.setOutputFilesLimit(metrics.outputFilesLimit());
metricsBuilder.setMemoryBytesLimit(metrics.memoryLimit());
if (metrics.timeLimitInMs() != 0L) {
metricsBuilder.setTimeLimit(millisToProto(metrics.timeLimitInMs()));
}

try (SilentCloseable c = Profiler.instance().profile("logSpawn/write")) {
Expand Down
26 changes: 14 additions & 12 deletions src/test/java/com/google/devtools/build/lib/exec/AbstractSpawnStrategyTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -331,7 +331,7 @@ public void testCacheMissWithNonZeroExit() throws Exception {

@Test
public void testLogSpawn() throws Exception {
setUpExecutionContext(/* executionOptions= */ null, /* remoteOptions= */ null);
setUpExecutionContext(/* remoteOptions= */ null);

Artifact input = ActionsTestUtil.createArtifact(rootDir, scratch.file("/execroot/foo", "1"));
scratch.file("/execroot/out1", "123");
Expand Down Expand Up @@ -402,13 +402,14 @@ public void testLogSpawn() throws Exception {
.setRunner("runner")
.setWalltime(Duration.getDefaultInstance())
.setTargetLabel("//dummy:label")
.setMetrics(Protos.SpawnMetrics.getDefaultInstance())
.build();
verify(messageOutput).write(expectedSpawnLog);
}

@Test
public void testLogSpawn_noPlatform_noLoggedPlatform() throws Exception {
setUpExecutionContext(/* executionOptions= */ null, /* remoteOptions= */ null);
setUpExecutionContext(/* remoteOptions= */ null);

Spawn spawn = new SpawnBuilder("cmd").build();

Expand All @@ -435,7 +436,7 @@ public void testLogSpawn_defaultPlatform_getsLogged() throws Exception {
" value: \"1\"",
"}");

setUpExecutionContext(/* executionOptions= */ null, remoteOptions);
setUpExecutionContext(remoteOptions);
Spawn spawn = new SpawnBuilder("cmd").build();
assertThrows(
SpawnExecException.class,
Expand All @@ -452,10 +453,7 @@ public void testLogSpawn_defaultPlatform_getsLogged() throws Exception {

@Test
public void testLogSpawn_spawnMetrics() throws Exception {
ExecutionOptions executionOptions = Options.getDefaults(ExecutionOptions.class);
executionOptions.executionLogSpawnMetrics = true;

setUpExecutionContext(executionOptions, /* remoteOptions= */ null);
setUpExecutionContext(/* remoteOptions= */ null);

assertThrows(
SpawnExecException.class,
Expand All @@ -480,7 +478,7 @@ public void testLogSpawn_specifiedPlatform_overridesDefault() throws Exception {
" name: \"a\"",
" value: \"1\"",
"}");
setUpExecutionContext(/* executionOptions= */ null, remoteOptions);
setUpExecutionContext(remoteOptions);

PlatformInfo platformInfo =
PlatformInfo.builder()
Expand Down Expand Up @@ -514,14 +512,17 @@ public void testLogSpawn_specifiedPlatform_overridesDefault() throws Exception {
verify(messageOutput).write(expected); // output will reflect default properties
}

private void setUpExecutionContext(ExecutionOptions executionOptions, RemoteOptions remoteOptions)
throws Exception {
private void setUpExecutionContext(RemoteOptions remoteOptions) throws Exception {
when(actionExecutionContext.getContext(eq(SpawnCache.class))).thenReturn(SpawnCache.NO_CACHE);
when(actionExecutionContext.getExecRoot()).thenReturn(execRoot);
when(actionExecutionContext.getContext(eq(SpawnLogContext.class)))
.thenReturn(
new SpawnLogContext(
execRoot, messageOutput, executionOptions, remoteOptions, SyscallCache.NO_CACHE));
execRoot,
messageOutput,
Options.getDefaults(ExecutionOptions.class),
remoteOptions,
SyscallCache.NO_CACHE));
when(spawnRunner.exec(any(Spawn.class), any(SpawnExecutionContext.class)))
.thenReturn(
new SpawnResult.Builder()
Expand All @@ -547,6 +548,7 @@ private static SpawnExec.Builder defaultSpawnExecBuilder(String cmd) {
.setExitCode(23)
.setRemoteCacheable(true)
.setWalltime(Duration.getDefaultInstance())
.setTargetLabel("//dummy:label");
.setTargetLabel("//dummy:label")
.setMetrics(Protos.SpawnMetrics.getDefaultInstance());
}
}

0 comments on commit c0f2e14

Please sign in to comment.