Merge branch 'main' into emeritus-2022-04

hyperledger · Apr 26, 2022 · 6b6a7b8 · 6b6a7b8
2 parents 12dac98 + c0dafd6
commit 6b6a7b8
Show file tree

Hide file tree

Showing 59 changed files with 1,280 additions and 238 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,9 +7,13 @@
 
 ## 22.4.0-RC2
 
+### Breaking Changes
+- In the Besu EVM Library all references to SHA3 have been renamed to the more accurate name Kecack256, including class names and comment. [#3749](https://github.com/hyperledger/besu/pull/3749)
+
 ### Additions and Improvements
 - Onchain node permissioning - log the enodeURL that was previously only throwing an IllegalStateException during the isPermitted check [#3697](https://github.com/hyperledger/besu/pull/3697)
 - \[EXPERIMENTAL\] Add snapsync `--sync-mode="X_SNAP"` (only as client) [#3710](https://github.com/hyperledger/besu/pull/3710)
+- Adapt Fast sync, and Snap sync, to use finalized block, from consensus layer, as pivot after the Merge [#3506](https://github.com/hyperledger/besu/issues/3506)
 
 ### Bug Fixes
 

diff --git a/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java b/besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java
@@ -17,6 +17,9 @@
 import static com.google.common.base.Preconditions.checkNotNull;
 
 import org.hyperledger.besu.config.GenesisConfigFile;
+import org.hyperledger.besu.config.GenesisConfigOptions;
+import org.hyperledger.besu.consensus.merge.FinalizedBlockHashSupplier;
+import org.hyperledger.besu.consensus.merge.MergeContext;
 import org.hyperledger.besu.consensus.qbft.pki.PkiBlockCreationConfiguration;
 import org.hyperledger.besu.crypto.NodeKey;
 import org.hyperledger.besu.datatypes.Hash;
@@ -49,8 +52,12 @@
 import org.hyperledger.besu.ethereum.eth.peervalidation.PeerValidator;
 import org.hyperledger.besu.ethereum.eth.peervalidation.RequiredBlocksPeerValidator;
 import org.hyperledger.besu.ethereum.eth.sync.DefaultSynchronizer;
+import org.hyperledger.besu.ethereum.eth.sync.PivotBlockSelector;
 import org.hyperledger.besu.ethereum.eth.sync.SyncMode;
 import org.hyperledger.besu.ethereum.eth.sync.SynchronizerConfiguration;
+import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromFinalizedBlock;
+import org.hyperledger.besu.ethereum.eth.sync.fastsync.PivotSelectorFromPeers;
+import org.hyperledger.besu.ethereum.eth.sync.fastsync.TransitionPivotSelector;
 import org.hyperledger.besu.ethereum.eth.sync.fullsync.SyncTerminationCondition;
 import org.hyperledger.besu.ethereum.eth.sync.state.SyncState;
 import org.hyperledger.besu.ethereum.eth.transactions.TransactionPool;
@@ -79,6 +86,7 @@
 import java.time.Clock;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -329,8 +337,9 @@ public BesuController build() {
             syncConfig.getComputationParallelism(),
             metricsSystem);
     final EthContext ethContext = new EthContext(ethPeers, ethMessages, snapMessages, scheduler);
-    final SyncState syncState = new SyncState(blockchain, ethPeers);
-    final boolean fastSyncEnabled = SyncMode.FAST.equals(syncConfig.getSyncMode());
+    final boolean fastSyncEnabled =
+        EnumSet.of(SyncMode.FAST, SyncMode.X_SNAP).contains(syncConfig.getSyncMode());
+    final SyncState syncState = new SyncState(blockchain, ethPeers, fastSyncEnabled);
 
     final TransactionPool transactionPool =
         TransactionPoolFactory.createTransactionPool(
@@ -360,6 +369,8 @@ public BesuController build() {
     final Optional<SnapProtocolManager> maybeSnapProtocolManager =
         createSnapProtocolManager(peerValidators, ethPeers, snapMessages, worldStateArchive);
 
+    final PivotBlockSelector pivotBlockSelector = createPivotSelector(protocolContext);
+
     final Synchronizer synchronizer =
         new DefaultSynchronizer(
             syncConfig,
@@ -368,12 +379,13 @@ public BesuController build() {
             worldStateStorage,
             ethProtocolManager.getBlockBroadcaster(),
             maybePruner,
-            ethProtocolManager.ethContext(),
+            ethContext,
             syncState,
             dataDirectory,
             clock,
             metricsSystem,
-            getFullSyncTerminationCondition(protocolContext.getBlockchain()));
+            getFullSyncTerminationCondition(protocolContext.getBlockchain()),
+            pivotBlockSelector);
 
     final MiningCoordinator miningCoordinator =
         createMiningCoordinator(
@@ -418,6 +430,41 @@ public BesuController build() {
         additionalPluginServices);
   }
 
+  private PivotBlockSelector createPivotSelector(final ProtocolContext protocolContext) {
+
+    final PivotSelectorFromPeers pivotSelectorFromPeers = new PivotSelectorFromPeers(syncConfig);
+    final GenesisConfigOptions genesisConfigOptions = genesisConfig.getConfigOptions();
+
+    if (genesisConfigOptions.getTerminalTotalDifficulty().isPresent()) {
+      LOG.info(
+          "TTD difficulty is present, creating initial sync phase with transition to PoS support");
+
+      final MergeContext mergeContext = protocolContext.getConsensusContext(MergeContext.class);
+      final FinalizedBlockHashSupplier finalizedBlockHashSupplier =
+          new FinalizedBlockHashSupplier();
+      final long subscriptionId =
+          mergeContext.addNewForkchoiceMessageListener(finalizedBlockHashSupplier);
+
+      final Runnable unsubscribeFinalizedBlockHashListener =
+          () -> {
+            mergeContext.removeNewForkchoiceMessageListener(subscriptionId);
+            LOG.info("Initial sync done, unsubscribe finalized block hash supplier");
+          };
+
+      return new TransitionPivotSelector(
+          genesisConfigOptions,
+          finalizedBlockHashSupplier,
+          pivotSelectorFromPeers,
+          new PivotSelectorFromFinalizedBlock(
+              genesisConfigOptions,
+              finalizedBlockHashSupplier,
+              unsubscribeFinalizedBlockHashListener));
+    } else {
+      LOG.info("TTD difficulty is not present, creating initial sync phase for PoW");
+      return pivotSelectorFromPeers;
+    }
+  }
+
   protected SyncTerminationCondition getFullSyncTerminationCondition(final Blockchain blockchain) {
     return genesisConfig
         .getConfigOptions()

diff --git a/.../merge/src/main/java/org/hyperledger/besu/consensus/merge/FinalizedBlockHashSupplier.java b/.../merge/src/main/java/org/hyperledger/besu/consensus/merge/FinalizedBlockHashSupplier.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright Hyperledger Besu Contributors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package org.hyperledger.besu.consensus.merge;
+
+import org.hyperledger.besu.consensus.merge.MergeContext.NewForkchoiceMessageListener;
+import org.hyperledger.besu.datatypes.Hash;
+
+import java.util.Optional;
+import java.util.function.Supplier;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FinalizedBlockHashSupplier
+    implements Supplier<Optional<Hash>>, NewForkchoiceMessageListener {
+  private static final Logger LOG = LoggerFactory.getLogger(FinalizedBlockHashSupplier.class);
+
+  private volatile Optional<Hash> lastAnnouncedFinalizedBlockHash = Optional.empty();
+
+  @Override
+  public void onNewForkchoiceMessage(
+      final Hash headBlockHash,
+      final Optional<Hash> maybeFinalizedBlockHash,
+      final Hash safeBlockHash) {
+    lastAnnouncedFinalizedBlockHash = maybeFinalizedBlockHash;
+    LOG.debug("New finalized block hash announced {}", lastAnnouncedFinalizedBlockHash);
+  }
+
+  @Override
+  public Optional<Hash> get() {
+    return lastAnnouncedFinalizedBlockHash;
+  }
+}
diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeContext.java
@@ -15,6 +15,7 @@
 package org.hyperledger.besu.consensus.merge;
 
 import org.hyperledger.besu.consensus.merge.blockcreation.PayloadIdentifier;
+import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.ConsensusContext;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
@@ -37,6 +38,11 @@ public interface MergeContext extends ConsensusContext {
 
   void observeNewIsPostMergeState(final NewMergeStateCallback newMergeStateCallback);
 
+  long addNewForkchoiceMessageListener(
+      final NewForkchoiceMessageListener newForkchoiceMessageListener);
+
+  void removeNewForkchoiceMessageListener(final long subscriberId);
+
   Difficulty getTerminalTotalDifficulty();
 
   void setFinalized(final BlockHeader blockHeader);
@@ -53,7 +59,19 @@ public interface MergeContext extends ConsensusContext {
 
   Optional<Block> retrieveBlockById(final PayloadIdentifier payloadId);
 
+  void fireNewForkchoiceMessageEvent(
+      final Hash headBlockHash,
+      final Optional<Hash> maybeFinalizedBlockHash,
+      final Hash safeBlockHash);
+
   interface NewMergeStateCallback {
     void onNewIsPostMergeState(final boolean newIsPostMergeState);
   }
+
+  interface NewForkchoiceMessageListener {
+    void onNewForkchoiceMessage(
+        final Hash headBlockHash,
+        final Optional<Hash> maybeFinalizedBlockHash,
+        final Hash safeBlockHash);
+  }
 }
diff --git a/...ensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeProtocolSchedule.java b/...ensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/MergeProtocolSchedule.java
@@ -23,9 +23,11 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecAdapters;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpecBuilder;
 import org.hyperledger.besu.ethereum.mainnet.feemarket.FeeMarket;
+import org.hyperledger.besu.evm.MainnetEVMs;
 import org.hyperledger.besu.evm.internal.EvmConfiguration;
 
 import java.math.BigInteger;
+import java.util.Optional;
 
 public class MergeProtocolSchedule {
 
@@ -44,7 +46,11 @@ public static ProtocolSchedule create(
     return new ProtocolScheduleBuilder(
             config,
             DEFAULT_CHAIN_ID,
-            ProtocolSpecAdapters.create(0, MergeProtocolSchedule::applyMergeSpecificModifications),
+            ProtocolSpecAdapters.create(
+                0,
+                (specBuilder) ->
+                    MergeProtocolSchedule.applyMergeSpecificModifications(
+                        specBuilder, config.getChainId())),
             privacyParameters,
             isRevertReasonEnabled,
             config.isQuorum(),
@@ -53,9 +59,13 @@ public static ProtocolSchedule create(
   }
 
   private static ProtocolSpecBuilder applyMergeSpecificModifications(
-      final ProtocolSpecBuilder specBuilder) {
+      final ProtocolSpecBuilder specBuilder, final Optional<BigInteger> chainId) {
 
     return specBuilder
+        .evmBuilder(
+            (gasCalculator, jdCacheConfig) ->
+                MainnetEVMs.paris(
+                    gasCalculator, chainId.orElse(BigInteger.ZERO), EvmConfiguration.DEFAULT))
         .blockProcessorBuilder(MergeBlockProcessor::new)
         .blockHeaderValidatorBuilder(MergeProtocolSchedule::getBlockHeaderValidator)
         .blockReward(Wei.ZERO)

diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/PostMergeContext.java
@@ -15,6 +15,7 @@
 package org.hyperledger.besu.consensus.merge;
 
 import org.hyperledger.besu.consensus.merge.blockcreation.PayloadIdentifier;
+import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.ConsensusContext;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
@@ -42,6 +43,8 @@ public class PostMergeContext implements MergeContext {
       new AtomicReference<>(Optional.empty());
   private final Subscribers<NewMergeStateCallback> newMergeStateCallbackSubscribers =
       Subscribers.create();
+  private final Subscribers<NewForkchoiceMessageListener> newForkchoiceMessageCallbackSubscribers =
+      Subscribers.create();
 
   private final EvictingQueue<PayloadTuple> blocksInProgress =
       EvictingQueue.create(MAX_BLOCKS_IN_PROGRESS);
@@ -123,6 +126,26 @@ public void observeNewIsPostMergeState(final NewMergeStateCallback newMergeState
     newMergeStateCallbackSubscribers.subscribe(newMergeStateCallback);
   }
 
+  @Override
+  public long addNewForkchoiceMessageListener(
+      final NewForkchoiceMessageListener newForkchoiceMessageListener) {
+    return newForkchoiceMessageCallbackSubscribers.subscribe(newForkchoiceMessageListener);
+  }
+
+  @Override
+  public void removeNewForkchoiceMessageListener(final long subscriberId) {
+    newForkchoiceMessageCallbackSubscribers.unsubscribe(subscriberId);
+  }
+
+  @Override
+  public void fireNewForkchoiceMessageEvent(
+      final Hash headBlockHash,
+      final Optional<Hash> maybeFinalizedBlockHash,
+      final Hash safeBlockHash) {
+    newForkchoiceMessageCallbackSubscribers.forEach(
+        cb -> cb.onNewForkchoiceMessage(headBlockHash, maybeFinalizedBlockHash, safeBlockHash));
+  }
+
   @Override
   public Difficulty getTerminalTotalDifficulty() {
     return terminalTotalDifficulty.get();

diff --git a/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java b/consensus/merge/src/main/java/org/hyperledger/besu/consensus/merge/TransitionContext.java
@@ -15,6 +15,7 @@
 package org.hyperledger.besu.consensus.merge;
 
 import org.hyperledger.besu.consensus.merge.blockcreation.PayloadIdentifier;
+import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.ConsensusContext;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.core.BlockHeader;
@@ -71,6 +72,26 @@ public void observeNewIsPostMergeState(final NewMergeStateCallback newMergeState
     postMergeContext.observeNewIsPostMergeState(newMergeStateCallback);
   }
 
+  @Override
+  public long addNewForkchoiceMessageListener(
+      final NewForkchoiceMessageListener newForkchoiceMessageListener) {
+    return postMergeContext.addNewForkchoiceMessageListener(newForkchoiceMessageListener);
+  }
+
+  @Override
+  public void removeNewForkchoiceMessageListener(final long subscriberId) {
+    postMergeContext.removeNewForkchoiceMessageListener(subscriberId);
+  }
+
+  @Override
+  public void fireNewForkchoiceMessageEvent(
+      final Hash headBlockHash,
+      final Optional<Hash> maybeFinalizedBlockHash,
+      final Hash safeBlockHash) {
+    postMergeContext.fireNewForkchoiceMessageEvent(
+        headBlockHash, maybeFinalizedBlockHash, safeBlockHash);
+  }
+
   @Override
   public Difficulty getTerminalTotalDifficulty() {
     return postMergeContext.getTerminalTotalDifficulty();

diff --git a/...s/merge/src/test/java/org/hyperledger/besu/consensus/merge/MergeProtocolScheduleTest.java b/...s/merge/src/test/java/org/hyperledger/besu/consensus/merge/MergeProtocolScheduleTest.java
@@ -18,10 +18,14 @@
 
 import org.hyperledger.besu.config.GenesisConfigFile;
 import org.hyperledger.besu.config.GenesisConfigOptions;
+import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
+import org.hyperledger.besu.evm.Code;
+import org.hyperledger.besu.evm.operation.PrevRanDaoOperation;
 
+import org.apache.tuweni.bytes.Bytes;
 import org.junit.Test;
 
 public class MergeProtocolScheduleTest {
@@ -55,5 +59,9 @@ public void parametersAlignWithMainnetWithAdjustments() {
     assertThat(london.getName()).isEqualTo("Frontier");
     assertThat(london.getBlockReward()).isEqualTo(Wei.ZERO);
     assertThat(london.isSkipZeroBlockRewards()).isEqualTo(true);
+
+    Bytes diffOp = Bytes.fromHexString("0x44");
+    var op = london.getEvm().operationAtOffset(Code.createLegacyCode(diffOp, Hash.hash(diffOp)), 0);
+    assertThat(op).isInstanceOf(PrevRanDaoOperation.class);
   }
 }
diff --git a/...yperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java b/...yperledger/besu/ethereum/api/jsonrpc/internal/methods/engine/EngineForkchoiceUpdated.java
@@ -66,6 +66,13 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     final Optional<EnginePayloadAttributesParameter> optionalPayloadAttributes =
         requestContext.getOptionalParameter(1, EnginePayloadAttributesParameter.class);
 
+    Optional<Hash> maybeFinalizedHash =
+        Optional.ofNullable(forkChoice.getFinalizedBlockHash())
+            .filter(finalized -> !Hash.ZERO.equals(finalized));
+
+    mergeContext.fireNewForkchoiceMessageEvent(
+        forkChoice.getHeadBlockHash(), maybeFinalizedHash, forkChoice.getSafeBlockHash());
+
     if (mergeContext.isSyncing()) {
       return new JsonRpcSuccessResponse(
           requestContext.getRequest().getId(),
@@ -80,14 +87,10 @@ public JsonRpcResponse syncResponse(final JsonRpcRequestContext requestContext)
     Optional<BlockHeader> newHead =
         protocolContext.getBlockchain().getBlockHeader(forkChoice.getHeadBlockHash());
 
-    Optional<Hash> finalizedHash =
-        Optional.ofNullable(forkChoice.getFinalizedBlockHash())
-            .filter(finalized -> !Hash.ZERO.equals(finalized));
-
     Optional<BlockHeader> finalizedHead =
-        finalizedHash.flatMap(protocolContext.getBlockchain()::getBlockHeader);
+        maybeFinalizedHash.flatMap(protocolContext.getBlockchain()::getBlockHeader);
 
-    if (newHead.isPresent() && (finalizedHash.isEmpty() || finalizedHead.isPresent())) {
+    if (newHead.isPresent() && (maybeFinalizedHash.isEmpty() || finalizedHead.isPresent())) {
 
       // TODO: post-merge cleanup, this should be unnecessary after merge
       if (!mergeCoordinator.latestValidAncestorDescendsFromTerminal(newHead.get())) {