do not disconnect a peer on re-pivot race (#3936)

* do not disconnect a peer on re-pivot race

Signed-off-by: garyschulte <garyschulte@gmail.com>

Co-authored-by: Sally MacFarlane <sally.macfarlane@consensys.net>

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/fastsync/FastSyncTargetManager.java

@@ -98,19 +98,27 @@ private CompletableFuture<Optional<EthPeer>> confirmPivotBlockHeader(final EthPe
     return ethContext
         .getScheduler()
         .timeout(task)
-        .thenApply(
+        .thenCompose(
             result -> {
               if (peerHasDifferentPivotBlock(result)) {
-                LOG.warn(
-                    "Best peer has wrong pivot block (#{}) expecting {} but received {}.  Disconnect: {}",
-                    pivotBlockHeader.getNumber(),
-                    pivotBlockHeader.getHash(),
-                    result.size() == 1 ? result.get(0).getHash() : "invalid response",
-                    bestPeer);
-                bestPeer.disconnect(DisconnectReason.USELESS_PEER);
-                return Optional.<EthPeer>empty();
+                if (!hasPivotChanged(pivotBlockHeader)) {
+                  // if the pivot block has not changed, then warn and disconnect this peer
+                  LOG.warn(
+                      "Best peer has wrong pivot block (#{}) expecting {} but received {}.  Disconnect: {}",
+                      pivotBlockHeader.getNumber(),
+                      pivotBlockHeader.getHash(),
+                      result.size() == 1 ? result.get(0).getHash() : "invalid response",
+                      bestPeer);
+                  bestPeer.disconnect(DisconnectReason.USELESS_PEER);
+                  return CompletableFuture.completedFuture(Optional.<EthPeer>empty());
+                }
+                LOG.debug(
+                    "Retrying best peer {} with new pivot block {}",
+                    bestPeer.getShortNodeId(),
+                    pivotBlockHeader.toLogString());
+                return confirmPivotBlockHeader(bestPeer);
               } else {
-                return Optional.of(bestPeer);
+                return CompletableFuture.completedFuture(Optional.of(bestPeer));
               }
             })
         .exceptionally(
@@ -120,6 +128,13 @@ private CompletableFuture<Optional<EthPeer>> confirmPivotBlockHeader(final EthPe
             });
   }
 
+  private boolean hasPivotChanged(final BlockHeader requestedPivot) {
+    return fastSyncState
+        .getPivotBlockHash()
+        .filter(currentPivotHash -> requestedPivot.getBlockHash().equals(currentPivotHash))
+        .isEmpty();
+  }
+
   private boolean peerHasDifferentPivotBlock(final List<BlockHeader> result) {
     final BlockHeader pivotBlockHeader = fastSyncState.getPivotBlockHeader().get();
     return result.size() != 1 || !result.get(0).equals(pivotBlockHeader);