Print remote execution message when the action times out (#15772)

Closes #15710. PiperOrigin-RevId: 456227285 Change-Id: I7d938bde6482151d96cb7cfe65242e659a6fb4a2 Co-authored-by: Chenchu K <ckolli@google.com>
bazelbuild · Jul 1, 2022 · ad17b44 · ad17b44
1 parent aedca1d
commit ad17b44
Show file tree

Hide file tree

Showing 2 changed files with 49 additions and 7 deletions.
diff --git a/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java b/src/main/java/com/google/devtools/build/lib/remote/RemoteSpawnRunner.java
@@ -272,11 +272,7 @@ public SpawnResult exec(Spawn spawn, SpawnExecutionContext context)
             // It's already late at this stage, but we should at least report once.
             reporter.reportExecutingIfNot();
 
-            FileOutErr outErr = context.getFileOutErr();
-            String message = result.getMessage();
-            if (!result.success() && !message.isEmpty()) {
-              outErr.printErr(message + "\n");
-            }
+            maybePrintExecutionMessages(context, result.getMessage(), result.success());
 
             profileAccounting(result.getExecutionMetadata());
             spawnMetricsAccounting(spawnMetrics, result.getExecutionMetadata());
@@ -444,6 +440,17 @@ public boolean handlesCaching() {
     return true;
   }
 
+  private void maybePrintExecutionMessages(
+      SpawnExecutionContext context, String message, boolean success) {
+    FileOutErr outErr = context.getFileOutErr();
+    boolean printMessage =
+        remoteOptions.remotePrintExecutionMessages.shouldPrintMessages(success)
+            && !message.isEmpty();
+    if (printMessage) {
+      outErr.printErr(message + "\n");
+    }
+  }
+
   private void maybeWriteParamFilesLocally(Spawn spawn) throws IOException {
     if (!executionOptions.shouldMaterializeParamFiles()) {
       return;
@@ -500,10 +507,11 @@ private SpawnResult execLocallyAndUploadOrFail(
     if (remoteOptions.remoteLocalFallback && !RemoteRetrierUtils.causedByExecTimeout(cause)) {
       return execLocallyAndUpload(action, spawn, context, uploadLocalResults);
     }
-    return handleError(action, cause);
+    return handleError(action, cause, context);
   }
 
-  private SpawnResult handleError(RemoteAction action, IOException exception)
+  private SpawnResult handleError(
+      RemoteAction action, IOException exception, SpawnExecutionContext context)
       throws ExecException, InterruptedException, IOException {
     boolean remoteCacheFailed =
         BulkTransferException.isOnlyCausedByCacheNotFoundException(exception);
@@ -522,6 +530,7 @@ private SpawnResult handleError(RemoteAction action, IOException exception)
         }
       }
       if (e.isExecutionTimeout()) {
+        maybePrintExecutionMessages(context, e.getResponse().getMessage(), /* success = */ false);
         return new SpawnResult.Builder()
             .setRunnerName(getName())
             .setStatus(Status.TIMEOUT)

diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java
@@ -566,6 +566,19 @@ public RemoteOutputsStrategyConverter() {
               + "that loads objects from the CAS on demand.")
   public String remoteDownloadSymlinkTemplate;
 
+  @Option(
+      name = "remote_print_execution_messages",
+      defaultValue = "failure",
+      converter = ExecutionMessagePrintMode.Converter.class,
+      category = "remote",
+      documentationCategory = OptionDocumentationCategory.LOGGING,
+      effectTags = {OptionEffectTag.TERMINAL_OUTPUT},
+      help =
+          "Choose when to print remote execution messages. Valid values are `failure`, "
+              + "to print only on failures, `success` to print only on successes and "
+              + "`all` to print always.")
+  public ExecutionMessagePrintMode remotePrintExecutionMessages;
+
   // The below options are not configurable by users, only tests.
   // This is part of the effort to reduce the overall number of flags.
 
@@ -635,4 +648,24 @@ private static FailureDetail createFailureDetail(String message, Code detailedCo
         .setRemoteExecution(RemoteExecution.newBuilder().setCode(detailedCode))
         .build();
   }
+
+  /** An enum for specifying different modes for printing remote execution messages. */
+  public enum ExecutionMessagePrintMode {
+    FAILURE, // Print execution messages only on failure
+    SUCCESS, // Print execution messages only on success
+    ALL; // Print execution messages always
+
+    /** Converts to {@link ExecutionMessagePrintMode}. */
+    public static class Converter extends EnumConverter<ExecutionMessagePrintMode> {
+      public Converter() {
+        super(ExecutionMessagePrintMode.class, "execution message print mode");
+      }
+    }
+
+    public boolean shouldPrintMessages(boolean success) {
+      return ((!success && this == ExecutionMessagePrintMode.FAILURE)
+          || (success && this == ExecutionMessagePrintMode.SUCCESS)
+          || this == ExecutionMessagePrintMode.ALL);
+    }
+  }
 }