Reduce space use spike after sync

src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs

@@ -36,7 +36,7 @@ public class DbConfig : IDbConfig
     public bool AllowMmapReads { get; set; } = false;
     public bool? VerifyChecksum { get; set; } = true;
     public double MaxBytesForLevelMultiplier { get; set; } = 10;
-    public ulong? MaxCompactionBytes { get; set; } = null;
+    public ulong? MaxCompactionBytes { get; set; } = (ulong)4.GiB();
     public int MinWriteBufferNumberToMerge { get; set; } = 1;
     public ulong? RowCacheSize { get; set; } = null;
     public bool OptimizeFiltersForHits { get; set; } = true;
@@ -52,6 +52,7 @@ public class DbConfig : IDbConfig
     public ulong BytesPerSync { get; set; } = 0;
     public double? DataBlockIndexUtilRatio { get; set; }
     public bool EnableFileWarmer { get; set; } = false;
+    public double CompressibilityHint { get; set; } = 1.0;
 
     public ulong BlobTransactionsDbBlockCacheSize { get; set; } = (ulong)32.MiB();
 
@@ -65,8 +66,9 @@ public class DbConfig : IDbConfig
     public bool? ReceiptsDbUseDirectReads { get; set; }
     public bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; }
     public ulong? ReceiptsDbCompactionReadAhead { get; set; }
-    public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)256.MiB();
-    public string? ReceiptsDbAdditionalRocksDbOptions { get; set; }
+    public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)64.MiB();
+    public double ReceiptsDbCompressibilityHint { get; set; } = 0.35;
+    public string? ReceiptsDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";
 
     public ulong BlocksDbWriteBufferSize { get; set; } = (ulong)64.MiB();
     public uint BlocksDbWriteBufferNumber { get; set; } = 2;
@@ -78,7 +80,7 @@ public class DbConfig : IDbConfig
     public bool? BlocksDbUseDirectReads { get; set; }
     public bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; }
     public ulong? BlocksDbCompactionReadAhead { get; set; }
-    public string? BlocksDbAdditionalRocksDbOptions { get; set; }
+    public string? BlocksDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";
 
     public ulong HeadersDbWriteBufferSize { get; set; } = (ulong)8.MiB();
     public uint HeadersDbWriteBufferNumber { get; set; } = 2;
@@ -90,7 +92,7 @@ public class DbConfig : IDbConfig
     public bool? HeadersDbUseDirectReads { get; set; }
     public bool? HeadersDbUseDirectIoForFlushAndCompactions { get; set; }
     public ulong? HeadersDbCompactionReadAhead { get; set; }
-    public string? HeadersDbAdditionalRocksDbOptions { get; set; }
+    public string? HeadersDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";
     public ulong? HeadersDbMaxBytesForLevelBase { get; set; } = (ulong)128.MiB();
 
     public ulong BlockNumbersDbWriteBufferSize { get; set; } = (ulong)8.MiB();
@@ -119,7 +121,7 @@ public class DbConfig : IDbConfig
     public bool? BlockInfosDbUseDirectReads { get; set; }
     public bool? BlockInfosDbUseDirectIoForFlushAndCompactions { get; set; }
     public ulong? BlockInfosDbCompactionReadAhead { get; set; }
-    public string? BlockInfosDbAdditionalRocksDbOptions { get; set; }
+    public string? BlockInfosDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";
 
     public ulong PendingTxsDbWriteBufferSize { get; set; } = (ulong)4.MiB();
     public uint PendingTxsDbWriteBufferNumber { get; set; } = 4;
@@ -202,6 +204,7 @@ public class DbConfig : IDbConfig
     public int? StateDbUseRibbonFilterStartingFromLevel { get; set; } = 2;
     public double? StateDbDataBlockIndexUtilRatio { get; set; } = 0.5;
     public bool StateDbEnableFileWarmer { get; set; } = false;
+    public double StateDbCompressibilityHint { get; set; } = 0.45;
     public string? StateDbAdditionalRocksDbOptions { get; set; }
 
     public uint RecycleLogFileNum { get; set; } = 0;

src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs

@@ -53,6 +53,7 @@ public interface IDbConfig : IConfig
     ulong BytesPerSync { get; set; }
     double? DataBlockIndexUtilRatio { get; set; }
     bool EnableFileWarmer { get; set; }
+    double CompressibilityHint { get; set; }
 
     ulong BlobTransactionsDbBlockCacheSize { get; set; }
 
@@ -67,6 +68,7 @@ public interface IDbConfig : IConfig
     bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; }
     ulong? ReceiptsDbCompactionReadAhead { get; set; }
     ulong ReceiptsDbTargetFileSizeBase { get; set; }
+    double ReceiptsDbCompressibilityHint { get; set; }
     string? ReceiptsDbAdditionalRocksDbOptions { get; set; }
 
     ulong BlocksDbWriteBufferSize { get; set; }
@@ -203,6 +205,7 @@ public interface IDbConfig : IConfig
     int? StateDbUseRibbonFilterStartingFromLevel { get; set; }
     double? StateDbDataBlockIndexUtilRatio { get; set; }
     bool StateDbEnableFileWarmer { get; set; }
+    double StateDbCompressibilityHint { get; set; }
     string? StateDbAdditionalRocksDbOptions { get; set; }
 
     /// <summary>

src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs

@@ -2,7 +2,6 @@
 // SPDX-License-Identifier: LGPL-3.0-only
 
 using System;
-using System.Collections.Generic;
 using System.IO;
 using System.Reflection;
 using Nethermind.Core.Extensions;
@@ -12,6 +11,7 @@ namespace Nethermind.Db.Rocks.Config;
 public class PerTableDbConfig
 {
     private readonly string _tableName;
+    private readonly string? _columnName;
     private readonly IDbConfig _dbConfig;
     private readonly DbSettings _settings;
 
@@ -20,10 +20,7 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum
         _dbConfig = dbConfig;
         _settings = dbSettings;
         _tableName = _settings.DbName;
-        if (columnName is not null)
-        {
-            _tableName += columnName;
-        }
+        _columnName = columnName;
     }
 
     public bool CacheIndexAndFilterBlocks => _settings.CacheIndexAndFilterBlocks ?? ReadConfig<bool>(nameof(CacheIndexAndFilterBlocks));
@@ -74,46 +71,73 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum
     public ulong BytesPerSync => ReadConfig<ulong>(nameof(BytesPerSync));
     public double? DataBlockIndexUtilRatio => ReadConfig<double?>(nameof(DataBlockIndexUtilRatio));
     public bool EnableFileWarmer => ReadConfig<bool>(nameof(EnableFileWarmer));
+    public double CompressibilityHint => ReadConfig<double>(nameof(CompressibilityHint));
 
     private T? ReadConfig<T>(string propertyName)
     {
-        return ReadConfig<T>(_dbConfig, propertyName, GetPrefix());
+        return ReadConfig<T>(_dbConfig, propertyName, GetPrefixes());
     }
 
-    private string GetPrefix()
+    private string[] GetPrefixes()
     {
-        return _tableName.StartsWith("State") ? "StateDb" : string.Concat(_tableName, "Db");
+        if (_tableName.StartsWith("State"))
+        {
+            return ["StateDb"];
+        }
+
+        if (_columnName != null)
+        {
+            return [
+                string.Concat(_tableName, _columnName, "Db"),
+                string.Concat(_tableName, "Db"),
+            ];
+        }
+
+        return [string.Concat(_tableName, "Db")];
     }
 
-    private static T? ReadConfig<T>(IDbConfig dbConfig, string propertyName, string prefix)
+    private static T? ReadConfig<T>(IDbConfig dbConfig, string propertyName, string[] prefixes)
     {
-        string prefixed = string.Concat(prefix, propertyName);
-
         try
         {
             Type type = dbConfig.GetType();
-            PropertyInfo? propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
+            PropertyInfo? propertyInfo;
 
-            if (propertyInfo is not null && propertyInfo.PropertyType.CanBeAssignedNull())
+            foreach (var prefix in prefixes)
             {
-                // If its nullable check if its null first
-                T? val = (T?)propertyInfo?.GetValue(dbConfig);
-                if (val is not null)
+                string prefixed = string.Concat(prefix, propertyName);
+
+                propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
+                if (propertyInfo is not null)
                 {
-                    return val;
+                    if (propertyInfo.PropertyType.CanBeAssignedNull())
+                    {
+                        // If its nullable check if its null first
+                        object? valObj = propertyInfo.GetValue(dbConfig);
+                        if (valObj is not null)
+                        {
+                            T? val = (T?)valObj;
+                            if (val is not null)
+                            {
+                                return val;
+                            }
+                        }
+                    }
+                    else
+                    {
+                        // If not nullable just use it directly
+                        return (T?)propertyInfo.GetValue(dbConfig);
+                    }
                 }
-
-                // Use generic one even if its available
-                propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
             }
 
-            // if no custom db property default to generic one
-            propertyInfo ??= type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
+            // Use generic one even if its available
+            propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
             return (T?)propertyInfo?.GetValue(dbConfig);
         }
         catch (Exception e)
         {
-            throw new InvalidDataException($"Unable to read {prefixed} property from DB config", e);
+            throw new InvalidDataException($"Unable to read property from DB config. Prefixes: ${prefixes}", e);
         }
     }
 }

src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs

@@ -1500,7 +1500,7 @@ public virtual void Tune(ITunableDb.TuneType type)
             case ITunableDb.TuneType.HeavyWrite:
                 // Compaction spikes are clear at this point. Will definitely affect attestation performance.
                 // Its unclear if it improve or slow down sync time. Seems to be the sweet spot.
-                ApplyOptions(GetHeavyWriteOptions((ulong)4.GiB()));
+                ApplyOptions(GetHeavyWriteOptions((ulong)2.GiB()));
                 break;
             case ITunableDb.TuneType.AggressiveHeavyWrite:
                 // For when, you are desperate, but don't wanna disable compaction completely, because you don't want
@@ -1609,8 +1609,10 @@ private IDictionary<string, string> GetHeavyWriteOptions(ulong l0SizeTarget)
         // but no io, only cpu.
         // bufferSize*maxBufferNumber = 128MB, which is the max memory used, which tend to be the case as its now
         // stalled by compaction instead of flush.
-        ulong bufferSize = (ulong)16.MiB();
-        ulong l0FileSize = bufferSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge;
+        // The buffer is not compressed unlike l0File, so to account for it, its size need to be slightly larger.
+        ulong targetFileSize = (ulong)16.MiB();
+        ulong bufferSize = (ulong)(targetFileSize / _perTableDbConfig.CompressibilityHint);
+        ulong l0FileSize = targetFileSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge;
         ulong maxBufferNumber = 8;
 
         // Guide recommend to have l0 and l1 to be the same size. They have to be compacted together so if l1 is larger,

src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs

@@ -38,6 +38,17 @@ public void CanReadAllConfigForAllTable()
         }
     }
 
+    [Test]
+    public void When_ColumnDb_UsePerTableConfig()
+    {
+        DbConfig dbConfig = new DbConfig();
+        dbConfig.MaxOpenFiles = 2;
+        dbConfig.ReceiptsDbMaxOpenFiles = 3;
+
+        PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, ""), "Blocks");
+        config.MaxOpenFiles.Should().Be(3);
+    }
+
     [Test]
     public void When_PerTableConfigIsAvailable_UsePerTableConfig()
     {