merged in changes from master. All tests passing

Signed-off-by: Stefan Pingel <stefan.pingel@consensys.net>

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -237,23 +237,24 @@ public BesuController build() {
     final ProtocolSchedule protocolSchedule = createProtocolSchedule();
     final GenesisState genesisState = GenesisState.fromConfig(genesisConfig, protocolSchedule);
     final WorldStateStorage worldStateStorage = storageProvider.createWorldStateStorage();
-    final WorldStateArchive worldStateArchive = createWorldStateArchive(worldStateStorage);
     final WorldStateStorage privateWorldStateStorage =
         storageProvider.createPrivateWorldStateStorage();
     final WorldStateArchive privateWorldStateArchive =
         createPrivateWorldStateArchive(privateWorldStateStorage);
 
+    final BlockchainStorage blockchainStorage =
+        storageProvider.createBlockchainStorage(protocolSchedule);
+
+    final MutableBlockchain blockchain =
+        DefaultBlockchain.createMutable(
+            genesisState.getBlock(), blockchainStorage, metricsSystem, reorgLoggingThreshold);
+
+    final WorldStateArchive worldStateArchive =
+        createWorldStateArchive(worldStateStorage, blockchain);
     final ProtocolContext protocolContext =
         ProtocolContext.init(
-            storageProvider,
-            worldStateArchive,
-            genesisState,
-            protocolSchedule,
-            metricsSystem,
-            this::createConsensusContext,
-            reorgLoggingThreshold);
+            blockchain, worldStateArchive, genesisState, this::createConsensusContext);
     protocolContext.setPrivateWorldStateArchive(privateWorldStateArchive);
-
     validateContext(protocolContext);
 
     protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor(

consensus/common/src/main/java/org/hyperledger/besu/consensus/common/bft/BftProtocolSchedule.java

@@ -18,7 +18,6 @@
 
 import org.hyperledger.besu.config.BftConfigOptions;
 import org.hyperledger.besu.config.GenesisConfigOptions;
-import org.hyperledger.besu.config.IbftConfigOptions;
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Wei;

consensus/ibftlegacy/src/main/java/org/hyperledger/besu/consensus/ibftlegacy/IbftProtocolSchedule.java

@@ -18,7 +18,6 @@
 
 import org.hyperledger.besu.config.BftConfigOptions;
 import org.hyperledger.besu.config.GenesisConfigOptions;
-import org.hyperledger.besu.config.IbftConfigOptions;
 import org.hyperledger.besu.ethereum.core.PrivacyParameters;
 import org.hyperledger.besu.ethereum.core.Wei;
 import org.hyperledger.besu.ethereum.mainnet.MainnetBlockBodyValidator;

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/MainnetBlockValidator.java

@@ -90,8 +90,10 @@ public Optional<BlockProcessingOutputs> validateAndProcessBlock(
     }
 
     final Optional<MutableWorldState> maybeWorldState =
-        context.getWorldStateArchive().getMutable(parentHeader.getStateRoot());
-    if (maybeWorldState.isEmpty()) {
+        context
+            .getWorldStateArchive()
+            .getMutable(parentHeader.getStateRoot(), parentHeader.getHash());
+    if (!maybeWorldState.isPresent()) {
       LOG.debug(
           "Unable to process block {} because parent world state {} is not available",
           block.getHeader().getNumber(),
@@ -127,6 +129,8 @@ public Optional<BlockProcessingOutputs> validateAndProcessBlock(
    */
   protected Result processBlock(
       final ProtocolContext context, final MutableWorldState worldState, final Block block) {
+    System.out.println("Using the MainnetBlockValidator");
+
     return blockProcessor.processBlock(context.getBlockchain(), worldState, block);
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/goquorum/GoQuorumBlockProcessor.java

@@ -171,7 +171,7 @@ public Result processBlock(
       return AbstractBlockProcessor.Result.failed();
     }
 
-    worldState.persist(blockHeader.getHash());
+    worldState.persist(blockHeader);
     privateWorldState.persist(null);
 
     privateStorageUpdater.putPrivateStateRootHashMapping(

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/goquorum/GoQuorumBlockValidator.java

@@ -57,34 +57,37 @@ public GoQuorumBlockValidator(
   protected Result processBlock(
       final ProtocolContext context, final MutableWorldState worldState, final Block block) {
 
+    System.out.println("Using the GoQuormBlockValidator");
     final MutableWorldState privateWorldState =
-        getPrivateWorldState(context, worldState.rootHash());
+        getPrivateWorldState(context, worldState.rootHash(), block.getHash());
 
     return ((GoQuorumBlockProcessor) blockProcessor)
         .processBlock(context.getBlockchain(), worldState, privateWorldState, block);
   }
 
   private MutableWorldState getPrivateWorldState(
-      final ProtocolContext context, final Hash publicWorldStateRootHash) {
+      final ProtocolContext context, final Hash worldStateRootHash, final Hash publicBlockHash) {
     final Hash privateStateRootHash =
         goQuorumPrivateStorage
-            .getPrivateStateRootHash(publicWorldStateRootHash)
+            .getPrivateStateRootHash(worldStateRootHash)
             .orElse(Hash.EMPTY_TRIE_HASH);
 
     final Optional<MutableWorldState> maybePrivateWorldState =
-        context.getPrivateWorldStateArchive().flatMap(p -> p.getMutable(privateStateRootHash));
+        context
+            .getPrivateWorldStateArchive()
+            .flatMap(p -> p.getMutable(privateStateRootHash, publicBlockHash));
     if (maybePrivateWorldState.isEmpty()) {
       LOG.debug(
-          "Private world state not available for public world state root hash {}",
-          publicWorldStateRootHash);
+          "Private world state not available for public world state root hash {}, public block hash {}",
+          worldStateRootHash,
+          publicBlockHash);
 
       /*
        This should never happen because privateStateRootResolver will either return a matching
        private world state root hash, or the hash for an empty world state (first private tx ever).
       */
       throw new IllegalStateException(
-          "Private world state not available for public world state root hash "
-              + publicWorldStateRootHash);
+          "Private world state not available for public world state root hash " + publicBlockHash);
     }
     return maybePrivateWorldState.get();
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/AbstractBlockProcessor.java

@@ -147,51 +147,55 @@ public AbstractBlockProcessor.Result processBlock(
           return AbstractBlockProcessor.Result.failed();
         }
 
-    final List<TransactionReceipt> receipts = new ArrayList<>();
-    long currentGasUsed = 0;
-    for (final Transaction transaction : transactions) {
-      if (!hasAvailableBlockBudget(blockHeader, transaction, currentGasUsed)) {
-        return AbstractBlockProcessor.Result.failed();
+        final WorldUpdater worldStateUpdater = worldState.updater();
+        final BlockHashLookup blockHashLookup = new BlockHashLookup(blockHeader, blockchain);
+        final Address miningBeneficiary =
+            miningBeneficiaryCalculator.calculateBeneficiary(blockHeader);
+
+        final TransactionProcessingResult result =
+            transactionProcessor.processTransaction(
+                blockchain,
+                worldStateUpdater,
+                blockHeader,
+                transaction,
+                miningBeneficiary,
+                OperationTracer.NO_TRACING,
+                blockHashLookup,
+                true,
+                TransactionValidationParams.processingBlock(),
+                privateMetadataUpdater);
+        if (result.isInvalid()) {
+          LOG.info(
+              "Block processing error: transaction invalid '{}'. Block {} Transaction {}",
+              result.getValidationResult().getInvalidReason(),
+              blockHeader.getHash().toHexString(),
+              transaction.getHash().toHexString());
+          return AbstractBlockProcessor.Result.failed();
+        }
+
+        worldStateUpdater.commit();
+
+        currentGasUsed += transaction.getGasLimit() - result.getGasRemaining();
+
+        final TransactionReceipt transactionReceipt =
+            transactionReceiptFactory.create(
+                transaction.getType(), result, worldState, currentGasUsed);
+        receipts.add(transactionReceipt);
       }
 
-      final WorldUpdater worldStateUpdater = worldState.updater();
-      final BlockHashLookup blockHashLookup = new BlockHashLookup(blockHeader, blockchain);
-      final Address miningBeneficiary =
-          miningBeneficiaryCalculator.calculateBeneficiary(blockHeader);
-
-      final TransactionProcessingResult result =
-          transactionProcessor.processTransaction(
-              blockchain,
-              worldStateUpdater,
-              blockHeader,
-              transaction,
-              miningBeneficiary,
-              OperationTracer.NO_TRACING,
-              blockHashLookup,
-              true,
-              TransactionValidationParams.processingBlock(),
-              privateMetadataUpdater);
-
-      if (result.isInvalid()) {
-        LOG.info(
-            "Block processing error: transaction invalid '{}'. Block {} Transaction {}",
-            result.getValidationResult().getInvalidReason(),
-            blockHeader.getHash().toHexString(),
-            transaction.getHash().toHexString());
+      if (!rewardCoinbase(worldState, blockHeader, ommers, skipZeroBlockRewards)) {
+        // no need to log, rewardCoinbase logs the error.
         return AbstractBlockProcessor.Result.failed();
       }
 
-      worldStateUpdater.commit();
-
-      currentGasUsed += transaction.getGasLimit() - result.getGasRemaining();
-
-      final TransactionReceipt transactionReceipt =
-          transactionReceiptFactory.create(
-              transaction.getType(), result, worldState, currentGasUsed);
-      receipts.add(transactionReceipt);
+      worldState.persist(blockHeader);
+      return AbstractBlockProcessor.Result.successful(receipts);
+    } finally {
+      globalProcessBlock.end();
     }
   }
 
+  // TODO Stefan: is this method used anywhere?
   protected boolean hasAvailableBlockBudget(
       final BlockHeader blockHeader, final Transaction transaction, final long currentGasUsed) {
     if (!gasBudgetCalculator.hasBudget(