From 0d700b3e63582d5eb2a0ba7fb4f49095a68a36be Mon Sep 17 00:00:00 2001 From: Amirul Ashraf Date: Thu, 6 Jun 2024 19:04:56 +0800 Subject: [PATCH] Reduce space use spike after sync (#7138) --- .../Nethermind.Db.Rocks/Config/DbConfig.cs | 15 ++-- .../Nethermind.Db.Rocks/Config/IDbConfig.cs | 3 + .../Config/PerTableDbConfig.cs | 70 +++++++++++++------ .../Nethermind.Db.Rocks/DbOnTheRocks.cs | 8 ++- .../Config/PerTableDbConfigTests.cs | 11 +++ 5 files changed, 75 insertions(+), 32 deletions(-) diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs index 0b133070801..b18a43b70b5 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs @@ -36,7 +36,7 @@ public class DbConfig : IDbConfig public bool AllowMmapReads { get; set; } = false; public bool? VerifyChecksum { get; set; } = true; public double MaxBytesForLevelMultiplier { get; set; } = 10; - public ulong? MaxCompactionBytes { get; set; } = null; + public ulong? MaxCompactionBytes { get; set; } = (ulong)4.GiB(); public int MinWriteBufferNumberToMerge { get; set; } = 1; public ulong? RowCacheSize { get; set; } = null; public bool OptimizeFiltersForHits { get; set; } = true; @@ -52,6 +52,7 @@ public class DbConfig : IDbConfig public ulong BytesPerSync { get; set; } = 0; public double? DataBlockIndexUtilRatio { get; set; } public bool EnableFileWarmer { get; set; } = false; + public double CompressibilityHint { get; set; } = 1.0; public ulong BlobTransactionsDbBlockCacheSize { get; set; } = (ulong)32.MiB(); @@ -65,8 +66,9 @@ public class DbConfig : IDbConfig public bool? ReceiptsDbUseDirectReads { get; set; } public bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? ReceiptsDbCompactionReadAhead { get; set; } - public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)256.MiB(); - public string? ReceiptsDbAdditionalRocksDbOptions { get; set; } + public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)64.MiB(); + public double ReceiptsDbCompressibilityHint { get; set; } = 0.35; + public string? ReceiptsDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong BlocksDbWriteBufferSize { get; set; } = (ulong)64.MiB(); public uint BlocksDbWriteBufferNumber { get; set; } = 2; @@ -78,7 +80,7 @@ public class DbConfig : IDbConfig public bool? BlocksDbUseDirectReads { get; set; } public bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? BlocksDbCompactionReadAhead { get; set; } - public string? BlocksDbAdditionalRocksDbOptions { get; set; } + public string? BlocksDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong HeadersDbWriteBufferSize { get; set; } = (ulong)8.MiB(); public uint HeadersDbWriteBufferNumber { get; set; } = 2; @@ -90,7 +92,7 @@ public class DbConfig : IDbConfig public bool? HeadersDbUseDirectReads { get; set; } public bool? HeadersDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? HeadersDbCompactionReadAhead { get; set; } - public string? HeadersDbAdditionalRocksDbOptions { get; set; } + public string? HeadersDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong? HeadersDbMaxBytesForLevelBase { get; set; } = (ulong)128.MiB(); public ulong BlockNumbersDbWriteBufferSize { get; set; } = (ulong)8.MiB(); @@ -119,7 +121,7 @@ public class DbConfig : IDbConfig public bool? BlockInfosDbUseDirectReads { get; set; } public bool? BlockInfosDbUseDirectIoForFlushAndCompactions { get; set; } public ulong? BlockInfosDbCompactionReadAhead { get; set; } - public string? BlockInfosDbAdditionalRocksDbOptions { get; set; } + public string? BlockInfosDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst"; public ulong PendingTxsDbWriteBufferSize { get; set; } = (ulong)4.MiB(); public uint PendingTxsDbWriteBufferNumber { get; set; } = 4; @@ -202,6 +204,7 @@ public class DbConfig : IDbConfig public int? StateDbUseRibbonFilterStartingFromLevel { get; set; } = 2; public double? StateDbDataBlockIndexUtilRatio { get; set; } = 0.5; public bool StateDbEnableFileWarmer { get; set; } = false; + public double StateDbCompressibilityHint { get; set; } = 0.45; public string? StateDbAdditionalRocksDbOptions { get; set; } public uint RecycleLogFileNum { get; set; } = 0; diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs index e16202cedd7..b19730fdf50 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs @@ -53,6 +53,7 @@ public interface IDbConfig : IConfig ulong BytesPerSync { get; set; } double? DataBlockIndexUtilRatio { get; set; } bool EnableFileWarmer { get; set; } + double CompressibilityHint { get; set; } ulong BlobTransactionsDbBlockCacheSize { get; set; } @@ -67,6 +68,7 @@ public interface IDbConfig : IConfig bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; } ulong? ReceiptsDbCompactionReadAhead { get; set; } ulong ReceiptsDbTargetFileSizeBase { get; set; } + double ReceiptsDbCompressibilityHint { get; set; } string? ReceiptsDbAdditionalRocksDbOptions { get; set; } ulong BlocksDbWriteBufferSize { get; set; } @@ -203,6 +205,7 @@ public interface IDbConfig : IConfig int? StateDbUseRibbonFilterStartingFromLevel { get; set; } double? StateDbDataBlockIndexUtilRatio { get; set; } bool StateDbEnableFileWarmer { get; set; } + double StateDbCompressibilityHint { get; set; } string? StateDbAdditionalRocksDbOptions { get; set; } /// diff --git a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs index b50ffb7b4e8..5ca7bb5a6de 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Generic; using System.IO; using System.Reflection; using Nethermind.Core.Extensions; @@ -12,6 +11,7 @@ namespace Nethermind.Db.Rocks.Config; public class PerTableDbConfig { private readonly string _tableName; + private readonly string? _columnName; private readonly IDbConfig _dbConfig; private readonly DbSettings _settings; @@ -20,10 +20,7 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum _dbConfig = dbConfig; _settings = dbSettings; _tableName = _settings.DbName; - if (columnName is not null) - { - _tableName += columnName; - } + _columnName = columnName; } public bool CacheIndexAndFilterBlocks => _settings.CacheIndexAndFilterBlocks ?? ReadConfig(nameof(CacheIndexAndFilterBlocks)); @@ -74,46 +71,73 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum public ulong BytesPerSync => ReadConfig(nameof(BytesPerSync)); public double? DataBlockIndexUtilRatio => ReadConfig(nameof(DataBlockIndexUtilRatio)); public bool EnableFileWarmer => ReadConfig(nameof(EnableFileWarmer)); + public double CompressibilityHint => ReadConfig(nameof(CompressibilityHint)); private T? ReadConfig(string propertyName) { - return ReadConfig(_dbConfig, propertyName, GetPrefix()); + return ReadConfig(_dbConfig, propertyName, GetPrefixes()); } - private string GetPrefix() + private string[] GetPrefixes() { - return _tableName.StartsWith("State") ? "StateDb" : string.Concat(_tableName, "Db"); + if (_tableName.StartsWith("State")) + { + return ["StateDb"]; + } + + if (_columnName != null) + { + return [ + string.Concat(_tableName, _columnName, "Db"), + string.Concat(_tableName, "Db"), + ]; + } + + return [string.Concat(_tableName, "Db")]; } - private static T? ReadConfig(IDbConfig dbConfig, string propertyName, string prefix) + private static T? ReadConfig(IDbConfig dbConfig, string propertyName, string[] prefixes) { - string prefixed = string.Concat(prefix, propertyName); - try { Type type = dbConfig.GetType(); - PropertyInfo? propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + PropertyInfo? propertyInfo; - if (propertyInfo is not null && propertyInfo.PropertyType.CanBeAssignedNull()) + foreach (var prefix in prefixes) { - // If its nullable check if its null first - T? val = (T?)propertyInfo?.GetValue(dbConfig); - if (val is not null) + string prefixed = string.Concat(prefix, propertyName); + + propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + if (propertyInfo is not null) { - return val; + if (propertyInfo.PropertyType.CanBeAssignedNull()) + { + // If its nullable check if its null first + object? valObj = propertyInfo.GetValue(dbConfig); + if (valObj is not null) + { + T? val = (T?)valObj; + if (val is not null) + { + return val; + } + } + } + else + { + // If not nullable just use it directly + return (T?)propertyInfo.GetValue(dbConfig); + } } - - // Use generic one even if its available - propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); } - // if no custom db property default to generic one - propertyInfo ??= type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); + // Use generic one even if its available + propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance); return (T?)propertyInfo?.GetValue(dbConfig); } catch (Exception e) { - throw new InvalidDataException($"Unable to read {prefixed} property from DB config", e); + throw new InvalidDataException($"Unable to read property from DB config. Prefixes: ${prefixes}", e); } } } diff --git a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs index 85133e0704e..8b5ac589718 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs @@ -1500,7 +1500,7 @@ public virtual void Tune(ITunableDb.TuneType type) case ITunableDb.TuneType.HeavyWrite: // Compaction spikes are clear at this point. Will definitely affect attestation performance. // Its unclear if it improve or slow down sync time. Seems to be the sweet spot. - ApplyOptions(GetHeavyWriteOptions((ulong)4.GiB())); + ApplyOptions(GetHeavyWriteOptions((ulong)2.GiB())); break; case ITunableDb.TuneType.AggressiveHeavyWrite: // For when, you are desperate, but don't wanna disable compaction completely, because you don't want @@ -1609,8 +1609,10 @@ private IDictionary GetHeavyWriteOptions(ulong l0SizeTarget) // but no io, only cpu. // bufferSize*maxBufferNumber = 128MB, which is the max memory used, which tend to be the case as its now // stalled by compaction instead of flush. - ulong bufferSize = (ulong)16.MiB(); - ulong l0FileSize = bufferSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge; + // The buffer is not compressed unlike l0File, so to account for it, its size need to be slightly larger. + ulong targetFileSize = (ulong)16.MiB(); + ulong bufferSize = (ulong)(targetFileSize / _perTableDbConfig.CompressibilityHint); + ulong l0FileSize = targetFileSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge; ulong maxBufferNumber = 8; // Guide recommend to have l0 and l1 to be the same size. They have to be compacted together so if l1 is larger, diff --git a/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs b/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs index 22f78e51939..d9d65b1d024 100644 --- a/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs @@ -38,6 +38,17 @@ public void CanReadAllConfigForAllTable() } } + [Test] + public void When_ColumnDb_UsePerTableConfig() + { + DbConfig dbConfig = new DbConfig(); + dbConfig.MaxOpenFiles = 2; + dbConfig.ReceiptsDbMaxOpenFiles = 3; + + PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, ""), "Blocks"); + config.MaxOpenFiles.Should().Be(3); + } + [Test] public void When_PerTableConfigIsAvailable_UsePerTableConfig() {