Replace caching store with row cache (#6405)

src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs

@@ -175,6 +175,7 @@ public class DbConfig : IDbConfig
     public bool? StateDbDisableCompression { get; set; } = false;
     public int StateDbTargetFileSizeMultiplier { get; set; } = 2;
     public IDictionary<string, string>? StateDbAdditionalRocksDbOptions { get; set; }
+    public ulong? StateDbRowCacheSize { get; set; }
 
     public uint RecycleLogFileNum { get; set; } = 0;
     public bool WriteAheadLogSync { get; set; } = false;

src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs

@@ -174,6 +174,7 @@ public interface IDbConfig : IConfig
     ulong? StateDbCompactionReadAhead { get; set; }
     bool? StateDbDisableCompression { get; set; }
     int StateDbTargetFileSizeMultiplier { get; set; }
+    ulong? StateDbRowCacheSize { get; set; }
     IDictionary<string, string>? StateDbAdditionalRocksDbOptions { get; set; }
 
     /// <summary>

src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs

@@ -51,6 +51,7 @@ public PerTableDbConfig(IDbConfig dbConfig, RocksDbSettings rocksDbSettings, str
     public ulong MaxBytesForLevelBase => ReadConfig<ulong>(nameof(MaxBytesForLevelBase));
     public ulong TargetFileSizeBase => ReadConfig<ulong>(nameof(TargetFileSizeBase));
     public int TargetFileSizeMultiplier => ReadConfig<int>(nameof(TargetFileSizeMultiplier));
+    public ulong? RowCacheSize => ReadConfig<ulong?>(nameof(RowCacheSize));
 
     private T? ReadConfig<T>(string propertyName)
     {

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -53,6 +53,7 @@ public class DbOnTheRocks : IDb, ITunableDb
     private long _maxThisDbSize;
 
     protected IntPtr? _cache = null;
+    protected IntPtr? _rowCache = null;
 
     private readonly RocksDbSettings _settings;
 
@@ -357,6 +358,12 @@ protected virtual void BuildOptions<T>(PerTableDbConfig dbConfig, Options<T> opt
         options.SetCreateIfMissing();
         options.SetAdviseRandomOnOpen(true);
 
+        if (dbConfig.RowCacheSize > 0)
+        {
+            _rowCache = RocksDbSharp.Native.Instance.rocksdb_cache_create_lru(new UIntPtr(dbConfig.RowCacheSize.Value));
+            _rocksDbNative.rocksdb_options_set_row_cache(options.Handle, _rowCache.Value);
+        }
+
         /*
          * Multi-Threaded Compactions
          * Compactions are needed to remove multiple copies of the same key that may occur if an application overwrites an existing key. Compactions also process deletions of keys. Compactions may occur in multiple threads if configured appropriately.
@@ -1051,6 +1058,11 @@ private void ReleaseUnmanagedResources()
             _rocksDbNative.rocksdb_cache_destroy(_cache.Value);
         }
 
+        if (_rowCache.HasValue)
+        {
+            _rocksDbNative.rocksdb_cache_destroy(_rowCache.Value);
+        }
+
         if (_rateLimiter.HasValue)
         {
             _rocksDbNative.rocksdb_ratelimiter_destroy(_rateLimiter.Value);

src/Nethermind/Nethermind.Init/InitializeStateDb.cs

@@ -76,12 +76,10 @@ public Task Execute(CancellationToken cancellationToken)
             setApi.WitnessRepository = NullWitnessCollector.Instance;
         }
 
-        CachingStore cachedStateDb = getApi.DbProvider.StateDb
-            .Cached(Trie.MemoryAllowance.TrieNodeCacheCount);
         IKeyValueStore codeDb = getApi.DbProvider.CodeDb
             .WitnessedBy(witnessCollector);
 
-        IKeyValueStoreWithBatching stateWitnessedBy = cachedStateDb.WitnessedBy(witnessCollector);
+        IKeyValueStoreWithBatching stateWitnessedBy = getApi.DbProvider.StateDb.WitnessedBy(witnessCollector);
         IPersistenceStrategy persistenceStrategy;
         IPruningStrategy pruningStrategy;
         if (pruningConfig.Mode.IsMemory())
@@ -132,7 +130,6 @@ public Task Execute(CancellationToken cancellationToken)
             IFullPruningDb fullPruningDb = (IFullPruningDb)getApi.DbProvider!.StateDb;
             fullPruningDb.PruningStarted += (_, args) =>
             {
-                cachedStateDb.PersistCache(args.Context);
                 trieStore.PersistCache(args.Context, args.Context.CancellationTokenSource.Token);
             };
         }

src/Nethermind/Nethermind.Init/MemoryHintMan.cs

@@ -64,7 +64,7 @@ public void SetMemoryAllowances(
                 AssignFastBlocksMemory(syncConfig);
                 _remainingMemory -= FastBlocksMemory;
                 if (_logger.IsInfo) _logger.Info($"  Fast blocks memory: {FastBlocksMemory / 1000 / 1000,5} MB");
-                AssignTrieCacheMemory();
+                AssignTrieCacheMemory(dbConfig);
                 _remainingMemory -= TrieCacheMemory;
                 if (_logger.IsInfo) _logger.Info($"  Trie memory:        {TrieCacheMemory / 1000 / 1000,5} MB");
                 UpdateDbConfig(cpuCount, syncConfig, dbConfig, initConfig);
@@ -104,10 +104,10 @@ private void SetupMallocOpts(IInitConfig initConfig)
         public long PeersMemory { get; private set; }
         public long TrieCacheMemory { get; private set; }
 
-        private void AssignTrieCacheMemory()
+        private void AssignTrieCacheMemory(IDbConfig dbConfig)
         {
             TrieCacheMemory = (long)(0.2 * _remainingMemory);
-            Trie.MemoryAllowance.TrieNodeCacheMemory = TrieCacheMemory;
+            dbConfig.StateDbRowCacheSize = (ulong)TrieCacheMemory;
         }
 
         private void AssignPeersMemory(INetworkConfig networkConfig)

src/Nethermind/Nethermind.Runner.Test/MemoryHintManTests.cs

@@ -168,7 +168,7 @@ public void Big_value_at_memory_hint(long memoryHint)
         {
             _initConfig.MemoryHint = memoryHint;
             SetMemoryAllowances(1);
-            Trie.MemoryAllowance.TrieNodeCacheCount.Should().BeGreaterThan(0);
+            _dbConfig.StateDbRowCacheSize.Should().BeGreaterThan(0);
         }
 
         [TestCase(true)]

src/Nethermind/Nethermind.State.Test/Witnesses/WitnessingStoreTests.cs

@@ -40,9 +40,9 @@ public void Does_not_collect_if_no_tracking()
         }
 
         [Test]
-        public void Collects_on_reads_when_cached_underneath()
+        public void Collects_on_reads_2()
         {
-            Context context = new(2);
+            Context context = new();
             context.Wrapped[Key1] = Value1;
             context.Wrapped[Key2] = Value2;
             context.Wrapped[Key3] = Value3;
@@ -63,9 +63,9 @@ public void Collects_on_reads_when_cached_underneath()
         }
 
         [Test]
-        public void Collects_on_reads_when_cached_underneath_and_previously_populated()
+        public void Collects_on_reads_and_previously_populated()
         {
-            Context context = new(3);
+            Context context = new();
 
             using IDisposable tracker = context.WitnessCollector.TrackOnThisThread();
             context.Database[Key1] = Value1;
@@ -113,12 +113,6 @@ public Context()
                 WitnessCollector = new WitnessCollector(new MemDb(), LimboLogs.Instance);
                 Database = new WitnessingStore(Wrapped, WitnessCollector);
             }
-
-            public Context(int cacheSize)
-            {
-                WitnessCollector = new WitnessCollector(new MemDb(), LimboLogs.Instance);
-                Database = new WitnessingStore(new CachingStore(Wrapped, cacheSize), WitnessCollector);
-            }
         }
 
         private static readonly byte[] Key1 = TestItem.KeccakA.BytesToArray();