Add --experimental_max_repeated_lost_inputs to make the action-rewinding lost-input limit configurable

src/main/java/com/google/devtools/build/lib/buildtool/BuildRequestOptions.java

@@ -379,6 +379,20 @@ public String getSymlinkPrefix(String productName) {
       help = "Whether to use action rewinding to recover from lost inputs.")
   public abstract boolean getRewindLostInputs();
 
+  // The default is kept in sync with ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS.
+  @Option(
+      name = "experimental_max_repeated_lost_inputs",
+      defaultValue = "20",
+      documentationCategory = OptionDocumentationCategory.REMOTE,
+      effectTags = {OptionEffectTag.EXECUTION},
+      help =
+          "The maximum number of times action rewinding will try to recover the same lost input (or"
+              + " top-level output) for the same action before giving up and failing the build."
+              + " Raise this to tolerate more transient remote-cache eviction of"
+              + " build-without-the-bytes outputs. Only takes effect when --rewind_lost_inputs is"
+              + " enabled.")
+  public abstract int getMaxRepeatedLostInputs();
+
   @Option(
       name = "incompatible_skip_genfiles_symlink",
       defaultValue = "true",

src/main/java/com/google/devtools/build/lib/skyframe/SkyframeActionExecutor.java

@@ -244,6 +244,7 @@ public void injectTree(SpecialArtifact output, TreeArtifactValue tree) {
   private OutputService outputService;
   private boolean finalizeActions;
   private boolean rewindingEnabled;
+  private int maxRepeatedLostInputs;
   private boolean invocationRetriesEnabled;
   private final Supplier<ImmutableList<Root>> sourceRootSupplier;
 
@@ -344,6 +345,7 @@ void prepareForExecution(
     // Cache some option values for performance, since we consult them on every action.
     this.finalizeActions = buildRequestOptions.getFinalizeActions();
     this.rewindingEnabled = buildRequestOptions.getRewindLostInputs();
+    this.maxRepeatedLostInputs = buildRequestOptions.getMaxRepeatedLostInputs();
     this.invocationRetriesEnabled =
         options.getOptions(ExecutionOptions.class).getRemoteRetryOnTransientCacheError() > 0;
     this.outputService = checkNotNull(outputService);
@@ -436,6 +438,15 @@ public boolean rewindingEnabled() {
     return rewindingEnabled;
   }
 
+  /**
+   * Returns the maximum number of times the same input (or top-level output) may be lost by the
+   * same action before rewinding gives up and fails the build. Configured by {@code
+   * --experimental_max_repeated_lost_inputs}.
+   */
+  public int maxRepeatedLostInputs() {
+    return maxRepeatedLostInputs;
+  }
+
   public boolean invocationRetriesEnabled() {
     return invocationRetriesEnabled;
   }

src/main/java/com/google/devtools/build/lib/skyframe/rewinding/ActionRewindStrategy.java

@@ -91,6 +91,9 @@
 public final class ActionRewindStrategy {
 
   private static final GoogleLogger logger = GoogleLogger.forEnclosingClass();
+  // Default for --experimental_max_repeated_lost_inputs, which overrides this value per build via
+  // SkyframeActionExecutor#maxRepeatedLostInputs. Retained as the historical default and test
+  // anchor.
   @VisibleForTesting static final int MAX_REPEATED_LOST_INPUTS = 20;
   @VisibleForTesting static final int MAX_ACTION_REWIND_EVENTS = 5;
   private static final int MAX_LOST_INPUTS_RECORDED = 5;
@@ -487,7 +490,7 @@ private void checkIfTopLevelOutputLostTooManyTimes(
     for (LostInputRecord lostInputRecord : currentAttemptLostOutputRecords) {
       String digest = lostInputRecord.lostInputDigest();
       int losses = historyForThisTopLevelKey.add(lostInputRecord, /* occurrences= */ 1) + 1;
-      if (losses > MAX_REPEATED_LOST_INPUTS) {
+      if (losses > skyframeActionExecutor.maxRepeatedLostInputs()) {
         ActionInput lostOutput =
             Iterables.find(
                 lostOutputsByDigest.get(digest),
@@ -496,7 +499,9 @@ private void checkIfTopLevelOutputLostTooManyTimes(
             new GenericActionRewindException(
                 String.format(
                     "Lost output %s (digest %s), and rewinding was ineffective after %d attempts.",
-                    prettyPrint(lostOutput), digest, MAX_REPEATED_LOST_INPUTS),
+                    prettyPrint(lostOutput),
+                    digest,
+                    skyframeActionExecutor.maxRepeatedLostInputs()),
                 ActionRewinding.Code.LOST_OUTPUT_TOO_MANY_TIMES);
         bugReporter.sendBugReport(e);
         throw e;
@@ -554,7 +559,7 @@ private void checkIfActionLostInputTooManyTimes(
       // the same input is repeatedly lost.
       String digest = lostInputRecord.lostInputDigest();
       int losses = historyForThisAction.add(lostInputRecord, /* occurrences= */ 1) + 1;
-      if (losses > MAX_REPEATED_LOST_INPUTS) {
+      if (losses > skyframeActionExecutor.maxRepeatedLostInputs()) {
         // This ensures coalesced shared actions aren't orphaned.
         skyframeActionExecutor.prepareForRewinding(
             failedKey, failedAction, /* depsToRewind= */ ImmutableList.of());

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTest.java

@@ -202,6 +202,12 @@ public void ineffectiveRewindingResultsInLostInputTooManyTimes() throws Exceptio
     assertOutputForRule2NotCreated();
   }
 
+  @Test
+  public void lostInputTooManyTimesLimitIsConfigurable() throws Exception {
+    helper.runLostInputTooManyTimesLimitIsConfigurable();
+    assertOutputForRule2NotCreated();
+  }
+
   @Test
   public void interruptedDuringRewindStopsNormally() throws Exception {
     helper.runInterruptedDuringRewindStopsNormally();

src/test/java/com/google/devtools/build/lib/skyframe/rewinding/RewindingTestsHelper.java

@@ -683,6 +683,57 @@ public final void runIneffectiveRewindingResultsInLostInputTooManyTimes() throws
         .isEqualTo(ActionRewindStrategy.MAX_REPEATED_LOST_INPUTS);
   }
 
+  /**
+   * Like {@link #runIneffectiveRewindingResultsInLostInputTooManyTimes}, but verifies that {@code
+   * --experimental_max_repeated_lost_inputs} overrides the default {@link
+   * ActionRewindStrategy#MAX_REPEATED_LOST_INPUTS} limit: rewinding gives up, and the build fails
+   * with {@link ActionRewinding.Code#LOST_INPUT_TOO_MANY_TIMES}, after the configured number of
+   * repeated losses rather than the default 20.
+   */
+  public final void runLostInputTooManyTimesLimitIsConfigurable() throws Exception {
+    int maxRepeatedLostInputs = 2;
+    testCase.addOptions("--experimental_max_repeated_lost_inputs=" + maxRepeatedLostInputs);
+    writeTwoGenrulePackage(testCase);
+
+    // Fail rule2 with the same lost input one more time than the configured limit allows.
+    AtomicReference<ActionInput> intermediate = new AtomicReference<>();
+    for (int i = 0; i <= maxRepeatedLostInputs; i++) {
+      addSpawnShim(
+          "Executing genrule //test:rule2",
+          (spawn, context) -> {
+            intermediate.set(SpawnInputUtils.getInputWithName(spawn, "intermediate.txt"));
+            return ExecResult.ofException(
+                new LostInputsExecException(
+                    ImmutableSetMultimap.of("fakedigest/10", intermediate.get())));
+          });
+    }
+
+    RecordingBugReporter bugReporter = testCase.recordBugReportsAndReinitialize();
+    BuildFailedException e =
+        assertThrows(BuildFailedException.class, () -> testCase.buildTarget("//test:rule2"));
+    assertThat(e.getDetailedExitCode().getFailureDetail().getActionRewinding().getCode())
+        .isEqualTo(ActionRewinding.Code.LOST_INPUT_TOO_MANY_TIMES);
+
+    // The give-up (and bug report) happens at the configured limit + 1, not the default 20 + 1.
+    String errorDetail =
+        String.format(
+            "lost input too many times (#%s) for the same action. lostInput: %s, "
+                + "lostInput digest: fakedigest/10, "
+                + "failedAction: action 'Executing genrule //test:rule2'",
+            maxRepeatedLostInputs + 1, intermediate.get());
+    assertThat(e.getDetailedExitCode().getFailureDetail().getMessage()).contains(errorDetail);
+    assertThat(Iterables.getOnlyElement(bugReporter.getExceptions()))
+        .hasMessageThat()
+        .contains(errorDetail);
+
+    // rule2 executed once per loss: the configured rewinds plus the final give-up attempt.
+    // With the default limit this would be 21; the configured limit of 2 makes it 3.
+    assertThat(
+            Iterables.frequency(
+                getExecutedSpawnDescriptions(), "Executing genrule //test:rule2"))
+        .isEqualTo(maxRepeatedLostInputs + 1);
+  }
+
   /**
    * Create N genrules that are dependent on a static source file. And then create another N
    * genrules that will consume the previous genrules equal to its index from 1 to N. For example,