Skip updating prewarm caches on main thread (#7203)

src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs

@@ -106,6 +106,7 @@ the previous head state.*/
         bool notReadOnly = !options.ContainsFlag(ProcessingOptions.ReadOnlyChain);
         int blocksCount = suggestedBlocks.Count;
         Block[] processedBlocks = new Block[blocksCount];
+
         try
         {
             for (int i = 0; i < blocksCount; i++)

src/Nethermind/Nethermind.Init/InitializeStateDb.cs

@@ -149,12 +149,16 @@ public Task Execute(CancellationToken cancellationToken)
                 mainWorldTrieStore,
                 codeDb,
                 getApi.LogManager,
-                preBlockCaches)
+                preBlockCaches,
+                // Main thread should only read from prewarm caches, not spend extra time updating them.
+                populatePreBlockCache: false)
             : new WorldState(
                 mainWorldTrieStore,
                 codeDb,
                 getApi.LogManager,
-                preBlockCaches);
+                preBlockCaches,
+                // Main thread should only read from prewarm caches, not spend extra time updating them.
+                populatePreBlockCache: false);
 
         // This is probably the point where a different state implementation would switch.
         IWorldStateManager stateManager = setApi.WorldStateManager = new WorldStateManager(

src/Nethermind/Nethermind.State/PersistentStorageProvider.cs

@@ -46,17 +46,19 @@ internal sealed class PersistentStorageProvider : PartialStorageProviderBase
         /// Manages persistent storage allowing for snapshotting and restoring
         /// Persists data to ITrieStore
         /// </summary>
-        public PersistentStorageProvider(ITrieStore? trieStore,
-            StateProvider? stateProvider,
-            ILogManager? logManager,
-            IStorageTreeFactory? storageTreeFactory = null,
-            ConcurrentDictionary<StorageCell, byte[]>? preBlockCache = null) : base(logManager)
+        public PersistentStorageProvider(ITrieStore trieStore,
+            StateProvider stateProvider,
+            ILogManager logManager,
+            IStorageTreeFactory? storageTreeFactory,
+            ConcurrentDictionary<StorageCell, byte[]>? preBlockCache,
+            bool populatePreBlockCache) : base(logManager)
         {
             _trieStore = trieStore ?? throw new ArgumentNullException(nameof(trieStore));
             _stateProvider = stateProvider ?? throw new ArgumentNullException(nameof(stateProvider));
             _logManager = logManager ?? throw new ArgumentNullException(nameof(logManager));
             _storageTreeFactory = storageTreeFactory ?? new StorageTreeFactory();
             _preBlockCache = preBlockCache;
+            _populatePreBlockCache = populatePreBlockCache;
             _loadFromTree = storageCell =>
             {
                 StorageTree tree = GetOrCreateStorage(storageCell.Address);
@@ -66,6 +68,7 @@ public PersistentStorageProvider(ITrieStore? trieStore,
         }
 
         public Hash256 StateRoot { get; set; } = null!;
+        private readonly bool _populatePreBlockCache;
 
         /// <summary>
         /// Reset the storage state
@@ -368,17 +371,9 @@ private ReadOnlySpan<byte> LoadFromTree(in StorageCell storageCell)
             ref byte[]? value = ref CollectionsMarshal.GetValueRefOrAddDefault(dict, storageCell.Index, out exists);
             if (!exists)
             {
-                long priorReads = Db.Metrics.ThreadLocalStorageTreeReads;
-
-                value = _preBlockCache is not null
-                    ? _preBlockCache.GetOrAdd(storageCell, _loadFromTree)
-                    : _loadFromTree(storageCell);
-
-                if (Db.Metrics.ThreadLocalStorageTreeReads == priorReads)
-                {
-                    // Read from Concurrent Cache
-                    Db.Metrics.IncrementStorageTreeCache();
-                }
+                value = !_populatePreBlockCache ?
+                    LoadFromTreeReadPreWarmCache(in storageCell) :
+                    LoadFromTreePopulatePrewarmCache(in storageCell);
             }
             else
             {
@@ -389,6 +384,35 @@ private ReadOnlySpan<byte> LoadFromTree(in StorageCell storageCell)
             return value;
         }
 
+        private byte[] LoadFromTreePopulatePrewarmCache(in StorageCell storageCell)
+        {
+            long priorReads = Db.Metrics.ThreadLocalStorageTreeReads;
+
+            byte[] value = _preBlockCache is not null
+                ? _preBlockCache.GetOrAdd(storageCell, _loadFromTree)
+                : _loadFromTree(storageCell);
+
+            if (Db.Metrics.ThreadLocalStorageTreeReads == priorReads)
+            {
+                // Read from Concurrent Cache
+                Db.Metrics.IncrementStorageTreeCache();
+            }
+            return value;
+        }
+
+        private byte[] LoadFromTreeReadPreWarmCache(in StorageCell storageCell)
+        {
+            if (_preBlockCache?.TryGetValue(storageCell, out byte[] value) ?? false)
+            {
+                Db.Metrics.IncrementStorageTreeCache();
+            }
+            else
+            {
+                value = _loadFromTree(storageCell);
+            }
+            return value;
+        }
+
         private void PushToRegistryOnly(in StorageCell cell, byte[] value)
         {
             StackList<int> stack = SetupRegistry(cell);

src/Nethermind/Nethermind.State/StateProvider.cs

@@ -44,6 +44,8 @@ internal class StateProvider
         internal readonly StateTree _tree;
         private readonly Func<AddressAsKey, Account> _getStateFromTrie;
 
+        private readonly bool _populatePreBlockCache;
+
         public void Accept(ITreeVisitor? visitor, Hash256? stateRoot, VisitingOptions? visitingOptions = null)
         {
             ArgumentNullException.ThrowIfNull(visitor);
@@ -640,12 +642,14 @@ private void ReportChanges(IStateTracer stateTracer, Dictionary<AddressAsKey, Ch
         }
 
         public StateProvider(IScopedTrieStore? trieStore,
-            IKeyValueStore? codeDb,
-            ILogManager? logManager,
+            IKeyValueStore codeDb,
+            ILogManager logManager,
             StateTree? stateTree = null,
-            ConcurrentDictionary<AddressAsKey, Account>? preBlockCache = null)
+            ConcurrentDictionary<AddressAsKey, Account>? preBlockCache = null,
+            bool populatePreBlockCache = true)
         {
             _preBlockCache = preBlockCache;
+            _populatePreBlockCache = populatePreBlockCache;
             _logger = logManager?.GetClassLogger<StateProvider>() ?? throw new ArgumentNullException(nameof(logManager));
             _codeDb = codeDb ?? throw new ArgumentNullException(nameof(codeDb));
             _tree = stateTree ?? new StateTree(trieStore, logManager);
@@ -667,15 +671,9 @@ public bool WarmUp(Address address)
             ref Account? account = ref CollectionsMarshal.GetValueRefOrAddDefault(_blockCache, addressAsKey, out bool exists);
             if (!exists)
             {
-                long priorReads = Metrics.ThreadLocalStateTreeReads;
-                account = _preBlockCache is not null
-                    ? _preBlockCache.GetOrAdd(addressAsKey, _getStateFromTrie)
-                    : _getStateFromTrie(addressAsKey);
-
-                if (Metrics.ThreadLocalStateTreeReads == priorReads)
-                {
-                    Metrics.IncrementStateTreeCacheHits();
-                }
+                account = !_populatePreBlockCache ?
+                    GetStateReadPreWarmCache(addressAsKey) :
+                    GetStatePopulatePrewarmCache(addressAsKey);
             }
             else
             {
@@ -684,6 +682,33 @@ public bool WarmUp(Address address)
             return account;
         }
 
+        private Account? GetStatePopulatePrewarmCache(AddressAsKey addressAsKey)
+        {
+            long priorReads = Metrics.ThreadLocalStateTreeReads;
+            Account? account = _preBlockCache is not null
+                ? _preBlockCache.GetOrAdd(addressAsKey, _getStateFromTrie)
+                : _getStateFromTrie(addressAsKey);
+
+            if (Metrics.ThreadLocalStateTreeReads == priorReads)
+            {
+                Metrics.IncrementStateTreeCacheHits();
+            }
+            return account;
+        }
+
+        private Account? GetStateReadPreWarmCache(AddressAsKey addressAsKey)
+        {
+            if (_preBlockCache?.TryGetValue(addressAsKey, out Account? account) ?? false)
+            {
+                Metrics.IncrementStateTreeCacheHits();
+            }
+            else
+            {
+                account = _getStateFromTrie(addressAsKey);
+            }
+            return account;
+        }
+
         private void SetState(Address address, Account? account)
         {
             _blockCache[address] = account;

src/Nethermind/Nethermind.State/WorldState.cs

@@ -54,17 +54,18 @@ internal WorldState(
             ILogManager? logManager,
             StateTree? stateTree = null,
             IStorageTreeFactory? storageTreeFactory = null,
-            PreBlockCaches? preBlockCaches = null)
+            PreBlockCaches? preBlockCaches = null,
+            bool populatePreBlockCache = true)
         {
             PreBlockCaches = preBlockCaches;
             _trieStore = trieStore;
-            _stateProvider = new StateProvider(trieStore.GetTrieStore(null), codeDb, logManager, stateTree, PreBlockCaches?.StateCache);
-            _persistentStorageProvider = new PersistentStorageProvider(trieStore, _stateProvider, logManager, storageTreeFactory, PreBlockCaches?.StorageCache);
+            _stateProvider = new StateProvider(trieStore.GetTrieStore(null), codeDb, logManager, stateTree, PreBlockCaches?.StateCache, populatePreBlockCache);
+            _persistentStorageProvider = new PersistentStorageProvider(trieStore, _stateProvider, logManager, storageTreeFactory, PreBlockCaches?.StorageCache, populatePreBlockCache);
             _transientStorageProvider = new TransientStorageProvider(logManager);
         }
 
-        public WorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager, PreBlockCaches? preBlockCaches)
-            : this(trieStore, codeDb, logManager, null, preBlockCaches: preBlockCaches)
+        public WorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager, PreBlockCaches? preBlockCaches, bool populatePreBlockCache = true)
+            : this(trieStore, codeDb, logManager, null, preBlockCaches: preBlockCaches, populatePreBlockCache: populatePreBlockCache)
         {
         }
 

src/Nethermind/Nethermind.Synchronization/Trie/HealingWorldState.cs

@@ -9,8 +9,8 @@
 
 namespace Nethermind.Synchronization.Trie;
 
-public class HealingWorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager, PreBlockCaches? preBlockCaches = null)
-    : WorldState(trieStore, codeDb, logManager, new HealingStateTree(trieStore, logManager), new HealingStorageTreeFactory(), preBlockCaches)
+public class HealingWorldState(ITrieStore trieStore, IKeyValueStore? codeDb, ILogManager? logManager, PreBlockCaches? preBlockCaches = null, bool populatePreBlockCache = true)
+    : WorldState(trieStore, codeDb, logManager, new HealingStateTree(trieStore, logManager), new HealingStorageTreeFactory(), preBlockCaches, populatePreBlockCache)
 {
     public void InitializeNetwork(ITrieNodeRecovery<GetTrieNodesRequest> recovery)
     {