Bonsai Tries Rolling Head and Layered Reads (#1750)

Two updates to Bonsai Tries

* Log Rolling is implemented on top of the existing Persisted head. When 
  Besu is at chain head and the new best head makes the current head an 
  orphan branch, the Bonsai TrieLogs are used to roll back to a common 
  block and roll forward to the needed base block. Goerli is known to 
  maintain sync. There are still some issues with frontier era block 
  receipts.
* Non-mutable reads can be done off of the persisted block. These are 
  accurate for all reads that were performed in the block. If a read is 
  not known it proceeds through a fallback series of calls to prior 
  layers until it hits the persisted block. These layered reads are 
  driven off of the TrieLogs.

Signed-off-by: Danno Ferrin <danno.ferrin@gmail.com>

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -26,6 +26,8 @@
 import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator;
 import org.hyperledger.besu.ethereum.bonsai.BonsaiWorldStateArchive;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
+import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
+import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
 import org.hyperledger.besu.ethereum.core.Hash;
@@ -230,23 +232,24 @@ public BesuController build() {
     final ProtocolSchedule protocolSchedule = createProtocolSchedule();
     final GenesisState genesisState = GenesisState.fromConfig(genesisConfig, protocolSchedule);
     final WorldStateStorage worldStateStorage = storageProvider.createWorldStateStorage();
-    final WorldStateArchive worldStateArchive = createWorldStateArchive(worldStateStorage);
+
+    final BlockchainStorage blockchainStorage =
+        storageProvider.createBlockchainStorage(protocolSchedule);
+
+    final MutableBlockchain blockchain =
+        DefaultBlockchain.createMutable(
+            genesisState.getBlock(), blockchainStorage, metricsSystem, reorgLoggingThreshold);
+
+    final WorldStateArchive worldStateArchive =
+        createWorldStateArchive(worldStateStorage, blockchain);
     final ProtocolContext protocolContext =
         ProtocolContext.init(
-            storageProvider,
-            worldStateArchive,
-            genesisState,
-            protocolSchedule,
-            metricsSystem,
-            this::createConsensusContext,
-            reorgLoggingThreshold);
+            blockchain, worldStateArchive, genesisState, this::createConsensusContext);
     validateContext(protocolContext);
 
     protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor(
         protocolContext.getWorldStateArchive());
 
-    final MutableBlockchain blockchain = protocolContext.getBlockchain();
-
     Optional<Pruner> maybePruner = Optional.empty();
     if (isPruningEnabled) {
       if (!storageProvider.isWorldStateIterable()) {
@@ -427,10 +430,11 @@ protected EthProtocolManager createEthProtocolManager(
         genesisConfig.getForks());
   }
 
-  public WorldStateArchive createWorldStateArchive(final WorldStateStorage worldStateStorage) {
+  private WorldStateArchive createWorldStateArchive(
+      final WorldStateStorage worldStateStorage, final Blockchain blockchain) {
     switch (dataStorageConfiguration.getDataStorageFormat()) {
       case BONSAI:
-        return new BonsaiWorldStateArchive(storageProvider);
+        return new BonsaiWorldStateArchive(storageProvider, blockchain);
       case FOREST:
       default:
         final WorldStatePreimageStorage preimageStorage =

ethereum/core/src/integration-test/java/org/hyperledger/besu/ethereum/vm/TraceTransactionIntegrationTest.java

@@ -163,7 +163,7 @@ public void shouldTraceContractCreation() {
     final Transaction transaction =
         Transaction.readFrom(
             new BytesValueRLPInput(Bytes.fromHexString(CONTRACT_CREATION_TX), false));
-    BlockHeader genesisBlockHeader = genesisBlock.getHeader();
+    final BlockHeader genesisBlockHeader = genesisBlock.getHeader();
     transactionProcessor.processTransaction(
         blockchain,
         worldStateArchive

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/ProtocolContext.java

@@ -15,14 +15,9 @@
 package org.hyperledger.besu.ethereum;
 
 import org.hyperledger.besu.ethereum.chain.Blockchain;
-import org.hyperledger.besu.ethereum.chain.BlockchainStorage;
-import org.hyperledger.besu.ethereum.chain.DefaultBlockchain;
 import org.hyperledger.besu.ethereum.chain.GenesisState;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
-import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
-import org.hyperledger.besu.ethereum.storage.StorageProvider;
 import org.hyperledger.besu.ethereum.worldstate.WorldStateArchive;
-import org.hyperledger.besu.plugin.services.MetricsSystem;
 
 import java.util.function.BiFunction;
 
@@ -46,20 +41,10 @@ public ProtocolContext(
   }
 
   public static ProtocolContext init(
-      final StorageProvider storageProvider,
+      final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
       final GenesisState genesisState,
-      final ProtocolSchedule protocolSchedule,
-      final MetricsSystem metricsSystem,
-      final BiFunction<Blockchain, WorldStateArchive, Object> consensusContextFactory,
-      final long reorgLoggingThreshold) {
-    final BlockchainStorage blockchainStorage =
-        storageProvider.createBlockchainStorage(protocolSchedule);
-
-    final MutableBlockchain blockchain =
-        DefaultBlockchain.createMutable(
-            genesisState.getBlock(), blockchainStorage, metricsSystem, reorgLoggingThreshold);
-
+      final BiFunction<Blockchain, WorldStateArchive, Object> consensusContextFactory) {
     if (blockchain.getChainHeadBlockNumber() < 1) {
       genesisState.writeStateTo(worldStateArchive.getMutable());
     }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiAccount.java

@@ -41,7 +41,7 @@
 import org.apache.tuweni.units.bigints.UInt256;
 
 public class BonsaiAccount implements MutableAccount, EvmAccount {
-  private final BonsaiWorldState context;
+  private final BonsaiWorldView context;
   private final boolean mutable;
 
   private final Address address;
@@ -56,7 +56,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   private final Map<UInt256, UInt256> updatedStorage = new HashMap<>();
 
   BonsaiAccount(
-      final BonsaiWorldState context,
+      final BonsaiWorldView context,
       final Address address,
       final Hash addressHash,
       final long nonce,
@@ -78,7 +78,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   }
 
   BonsaiAccount(
-      final BonsaiWorldState context,
+      final BonsaiWorldView context,
       final Address address,
       final StateTrieAccountValue stateTrieAccount,
       final boolean mutable) {
@@ -98,7 +98,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
     this(toCopy, toCopy.context, false);
   }
 
-  BonsaiAccount(final BonsaiAccount toCopy, final BonsaiWorldState context, final boolean mutable) {
+  BonsaiAccount(final BonsaiAccount toCopy, final BonsaiWorldView context, final boolean mutable) {
     this.context = context;
     this.address = toCopy.address;
     this.addressHash = toCopy.addressHash;
@@ -113,8 +113,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
     this.mutable = mutable;
   }
 
-  BonsaiAccount(
-      final BonsaiWorldState context, final UpdateTrackingAccount<BonsaiAccount> tracked) {
+  BonsaiAccount(final BonsaiWorldView context, final UpdateTrackingAccount<BonsaiAccount> tracked) {
     this.context = context;
     this.address = tracked.getAddress();
     this.addressHash = tracked.getAddressHash();
@@ -130,7 +129,7 @@ public class BonsaiAccount implements MutableAccount, EvmAccount {
   }
 
   static BonsaiAccount fromRLP(
-      final BonsaiWorldState context,
+      final BonsaiWorldView context,
       final Address address,
       final Bytes encoded,
       final boolean mutable)
@@ -202,7 +201,7 @@ public void setBalance(final Wei value) {
   @Override
   public Bytes getCode() {
     if (code == null) {
-      code = context.getCode(address);
+      code = context.getCode(address).orElse(Bytes.EMPTY);
     }
     return code;
   }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java

@@ -16,12 +16,11 @@
 
 package org.hyperledger.besu.ethereum.bonsai;
 
-import static com.google.common.base.Preconditions.checkArgument;
-
 import org.hyperledger.besu.ethereum.core.Account;
 import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.ethereum.core.Hash;
 import org.hyperledger.besu.ethereum.core.WorldState;
+import org.hyperledger.besu.ethereum.worldstate.StateTrieAccountValue;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -33,21 +32,73 @@
 import org.apache.tuweni.units.bigints.UInt256;
 
 /** A World State backed first by trie log layer and then by another world state. */
-public class BonsaiLayeredWorldState implements BonsaiWorldState, WorldState {
+public class BonsaiLayeredWorldState implements BonsaiWorldView, WorldState {
+
+  private final BonsaiWorldView parent;
+  protected final long height;
+  protected final TrieLogLayer trieLog;
 
-  private final BonsaiWorldState parent;
-  private final TrieLogLayer trieLog;
-  private final Map<Address, Account> cachedAccounts = new HashMap<>();
+  private final Hash worldStateRootHash;
 
-  public BonsaiLayeredWorldState(final BonsaiWorldState parent, final TrieLogLayer trieLog) {
-    checkArgument(trieLog.isFrozen(), "TrieLogs must be frozen to be used as world state.");
+  BonsaiLayeredWorldState(
+      final BonsaiWorldView parent,
+      final long height,
+      final Hash worldStateRootHash,
+      final TrieLogLayer trieLog) {
     this.parent = parent;
+    this.height = height;
+    this.worldStateRootHash = worldStateRootHash;
     this.trieLog = trieLog;
   }
 
+  public BonsaiWorldView getParent() {
+    return parent;
+  }
+
+  public TrieLogLayer getTrieLog() {
+    return trieLog;
+  }
+
+  public long getHeight() {
+    return height;
+  }
+
+  @Override
+  public Optional<Bytes> getCode(final Address address) {
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      final Optional<Bytes> maybeCode = currentLayer.trieLog.getCode(address);
+      if (maybeCode.isPresent()) {
+        return maybeCode;
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.getCode(address);
+      }
+    }
+    return Optional.empty();
+  }
+
   @Override
-  public Bytes getCode(final Address address) {
-    return trieLog.getCode(address).orElseGet(() -> parent.getCode(address));
+  public Optional<Bytes> getStateTrieNode(final Bytes location) {
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.getStateTrieNode(location);
+      }
+    }
+    return Optional.empty();
   }
 
   @Override
@@ -57,9 +108,24 @@ public UInt256 getStorageValue(final Address address, final UInt256 key) {
 
   @Override
   public Optional<UInt256> getStorageValueBySlotHash(final Address address, final Hash slotHash) {
-    return trieLog
-        .getStorageBySlotHash(address, slotHash)
-        .or(() -> parent.getStorageValueBySlotHash(address, slotHash));
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      final Optional<UInt256> maybeValue =
+          currentLayer.trieLog.getStorageBySlotHash(address, slotHash);
+      if (maybeValue.isPresent()) {
+        return maybeValue;
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.getStorageValueBySlotHash(address, slotHash);
+      }
+    }
+    return Optional.empty();
   }
 
   @Override
@@ -70,30 +136,75 @@ public UInt256 getOriginalStorageValue(final Address address, final UInt256 key)
 
   @Override
   public Map<Bytes32, Bytes> getAllAccountStorage(final Address address, final Hash rootHash) {
-    final Map<Bytes32, Bytes> results = parent.getAllAccountStorage(address, rootHash);
-    trieLog
-        .streamStorageChanges(address)
-        .forEach(entry -> results.put(entry.getKey(), entry.getValue().getUpdated().toBytes()));
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    final Map<Bytes32, Bytes> results = new HashMap<>();
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      if (currentLayer.trieLog.hasStorageChanges(address)) {
+        currentLayer
+            .trieLog
+            .streamStorageChanges(address)
+            .forEach(
+                entry -> {
+                  if (!results.containsKey(entry.getKey())) {
+                    final UInt256 value = entry.getValue().getUpdated();
+                    // yes, store the nulls.  If it was deleted it should stay deleted
+                    results.put(entry.getKey(), value == null ? null : value.toBytes());
+                  }
+                });
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        final Account account = currentLayer.parent.get(address);
+        if (account != null) {
+          account
+              .storageEntriesFrom(Hash.ZERO, Integer.MAX_VALUE)
+              .forEach(
+                  (k, v) -> {
+                    if (!results.containsKey(k)) {
+                      results.put(k, v.getValue().toBytes());
+                    }
+                  });
+        }
+        currentLayer = null;
+      }
+    }
     return results;
   }
 
   @Override
   public Account get(final Address address) {
-    return cachedAccounts.computeIfAbsent(
-        address,
-        addr ->
-            trieLog
-                .getAccount(addr)
-                .map(
-                    stateTrieAccountValue ->
-                        (Account)
-                            new BonsaiAccount(
-                                BonsaiLayeredWorldState.this, addr, stateTrieAccountValue, false))
-                .orElseGet(() -> parent.get(address)));
+    // this must be iterative and lambda light because the stack may blow up
+    // mainly because we don't have tail calls.
+    BonsaiLayeredWorldState currentLayer = this;
+    while (currentLayer != null) {
+      final Optional<StateTrieAccountValue> maybeStateTrieAccount =
+          currentLayer.trieLog.getAccount(address);
+      if (maybeStateTrieAccount.isPresent()) {
+        return new BonsaiAccount(
+            BonsaiLayeredWorldState.this, address, maybeStateTrieAccount.get(), false);
+      }
+      if (currentLayer.parent == null) {
+        currentLayer = null;
+      } else if (currentLayer.parent instanceof BonsaiLayeredWorldState) {
+        currentLayer = (BonsaiLayeredWorldState) currentLayer.parent;
+      } else {
+        return currentLayer.parent.get(address);
+      }
+    }
+    return null;
   }
 
   @Override
   public Hash rootHash() {
+    return worldStateRootHash;
+  }
+
+  public Hash blockHash() {
     return trieLog.getBlockHash();
   }