Fix Layered World State issue (#5076)

* Use the copy during prepareTrieLog instead of saveTrieLog
* add final flag on BonsaiWorldStateUpdater
* Use a copy of BonsaiInMemoryWorldState inside prepareTrieLog
* add link to persisted worldstate storage
* fix tests
* Make a copy of the worldstate after committing changes to the trielog
* spotless + remove maybeUnSubscribe in setNextWorldView
* subscribe storage on layered worldstate
* fix null issue
* not close layered worldstate during getAccount
* clean code
* Add changelog entry

---------

Signed-off-by: Ameziane H <ameziane.hamlat@consensys.net>
Signed-off-by: Karim TAAM <karim.t2am@gmail.com>
Co-authored-by: Karim TAAM <karim.t2am@gmail.com>
Signed-off-by: garyschulte <garyschulte@gmail.com>

CHANGELOG.md

@@ -28,7 +28,7 @@ tests are updated to use EC private keys instead of RSA keys.
 
 ### Bug Fixes
 - Mitigation fix for stale bonsai code storage leading to log rolling issues on contract recreates [#4906](https://github.com/hyperledger/besu/pull/4906)
-
+- Ensure latest cached layered worldstate is subscribed to storage, fix problem with RPC calls using 'latest' [#5076](https://github.com/hyperledger/besu/pull/5076)
 
 ## 23.1.0-RC1
 

ethereum/api/src/main/java/org/hyperledger/besu/ethereum/api/query/BlockchainQueries.java

@@ -855,12 +855,19 @@ public <U> Optional<U> getAndMapWorldState(
         .getBlockHeader(blockHash)
         .flatMap(
             blockHeader -> {
-              try (var ws =
+              try (final var worldState =
                   worldStateArchive
                       .getMutable(blockHeader.getStateRoot(), blockHeader.getHash(), false)
+                      .map(
+                          ws -> {
+                            if (!ws.isPersistable()) {
+                              return ws.copy();
+                            }
+                            return ws;
+                          })
                       .orElse(null)) {
-                if (ws != null) {
-                  return Optional.ofNullable(mapper.apply(ws));
+                if (worldState != null) {
+                  return Optional.ofNullable(mapper.apply(worldState));
                 }
               } catch (Exception ex) {
                 LOG.error("failed worldstate query for " + blockHash.toShortHexString(), ex);

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/AbstractTrieLogManager.java

@@ -97,16 +97,17 @@ protected abstract void addCachedLayer(
 
   @VisibleForTesting
   TrieLogLayer prepareTrieLog(
-      final BlockHeader blockHeader,
-      final Hash worldStateRootHash,
+      final BlockHeader forBlockHeader,
+      final Hash forWorldStateRootHash,
       final BonsaiWorldStateUpdater localUpdater,
       final BonsaiWorldStateArchive worldStateArchive,
       final BonsaiPersistedWorldState forWorldState) {
-    debugLambda(LOG, "Adding layered world state for {}", blockHeader::toLogString);
-    final TrieLogLayer trieLog = localUpdater.generateTrieLog(blockHeader.getBlockHash());
+    debugLambda(LOG, "Adding layered world state for {}", forBlockHeader::toLogString);
+    final TrieLogLayer trieLog = localUpdater.generateTrieLog(forBlockHeader.getBlockHash());
     trieLog.freeze();
-    addCachedLayer(blockHeader, worldStateRootHash, trieLog, worldStateArchive, forWorldState);
-    scrubCachedLayers(blockHeader.getNumber());
+    addCachedLayer(
+        forBlockHeader, forWorldStateRootHash, trieLog, worldStateArchive, forWorldState);
+    scrubCachedLayers(forBlockHeader.getNumber());
     return trieLog;
   }
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiLayeredWorldState.java

@@ -40,8 +40,13 @@
 import org.apache.tuweni.units.bigints.UInt256;
 
 /** A World State backed first by trie log layer and then by another world state. */
-public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldView, WorldState {
-  private Optional<BonsaiWorldView> nextWorldView;
+public class BonsaiLayeredWorldState
+    implements MutableWorldState,
+        BonsaiWorldView,
+        WorldState,
+        BonsaiWorldStateKeyValueStorage.BonsaiStorageSubscriber {
+  private Optional<BonsaiWorldView> nextWorldView = Optional.empty();
+  private Optional<Long> newtWorldViewSubscribeId = Optional.empty();
   protected final long height;
   protected final TrieLogLayer trieLog;
   private final Hash worldStateRootHash;
@@ -58,7 +63,7 @@ public class BonsaiLayeredWorldState implements MutableWorldState, BonsaiWorldVi
       final TrieLogLayer trieLog) {
     this.blockchain = blockchain;
     this.archive = archive;
-    this.nextWorldView = nextWorldView;
+    this.setNextWorldView(nextWorldView);
     this.height = height;
     this.worldStateRootHash = worldStateRootHash;
     this.trieLog = trieLog;
@@ -76,7 +81,34 @@ public Optional<BonsaiWorldView> getNextWorldView() {
   }
 
   public void setNextWorldView(final Optional<BonsaiWorldView> nextWorldView) {
+    maybeUnSubscribe(); // unsubscribe the old view
     this.nextWorldView = nextWorldView;
+    maybeSubscribe(); // subscribe the next view
+  }
+
+  private void maybeSubscribe() {
+    nextWorldView
+        .filter(BonsaiPersistedWorldState.class::isInstance)
+        .map(BonsaiPersistedWorldState.class::cast)
+        .ifPresent(
+            worldState -> {
+              newtWorldViewSubscribeId = Optional.of(worldState.worldStateStorage.subscribe(this));
+            });
+  }
+
+  private void maybeUnSubscribe() {
+    nextWorldView
+        .filter(BonsaiPersistedWorldState.class::isInstance)
+        .map(BonsaiPersistedWorldState.class::cast)
+        .ifPresent(
+            worldState -> {
+              newtWorldViewSubscribeId.ifPresent(worldState.worldStateStorage::unSubscribe);
+            });
+  }
+
+  @Override
+  public void close() throws Exception {
+    maybeUnSubscribe();
   }
 
   public TrieLogLayer getTrieLog() {

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/transactions/sorter/AbstractPendingTransactionsSorter.java

@@ -428,7 +428,11 @@ private TransactionAddedStatus addTransaction(
             LOG,
             "Transaction {} not added because nonce too far in the future for sender {}",
             transaction::toTraceLog,
-            maybeSenderAccount::toString);
+            () ->
+                maybeSenderAccount
+                    .map(Account::getAddress)
+                    .map(Address::toString)
+                    .orElse("unknown"));
         return NONCE_TOO_FAR_IN_FUTURE_FOR_SENDER;
       }