Skip to content

Commit

Permalink
Reduce space use spike after sync (#7138)
Browse files Browse the repository at this point in the history
  • Loading branch information
@asdacap
asdacap authored Jun 6, 2024
1 parent 0d7bbbe commit 0d700b3
Show file tree
Hide file tree
Showing 5 changed files with 75 additions and 32 deletions.
15 changes: 9 additions & 6 deletions src/Nethermind/Nethermind.Db.Rocks/Config/DbConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ public class DbConfig : IDbConfig
public bool AllowMmapReads { get; set; } = false;
public bool? VerifyChecksum { get; set; } = true;
public double MaxBytesForLevelMultiplier { get; set; } = 10;
public ulong? MaxCompactionBytes { get; set; } = null;
public ulong? MaxCompactionBytes { get; set; } = (ulong)4.GiB();
public int MinWriteBufferNumberToMerge { get; set; } = 1;
public ulong? RowCacheSize { get; set; } = null;
public bool OptimizeFiltersForHits { get; set; } = true;
Expand All @@ -52,6 +52,7 @@ public class DbConfig : IDbConfig
public ulong BytesPerSync { get; set; } = 0;
public double? DataBlockIndexUtilRatio { get; set; }
public bool EnableFileWarmer { get; set; } = false;
public double CompressibilityHint { get; set; } = 1.0;

public ulong BlobTransactionsDbBlockCacheSize { get; set; } = (ulong)32.MiB();

Expand All @@ -65,8 +66,9 @@ public class DbConfig : IDbConfig
public bool? ReceiptsDbUseDirectReads { get; set; }
public bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; }
public ulong? ReceiptsDbCompactionReadAhead { get; set; }
public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)256.MiB();
public string? ReceiptsDbAdditionalRocksDbOptions { get; set; }
public ulong ReceiptsDbTargetFileSizeBase { get; set; } = (ulong)64.MiB();
public double ReceiptsDbCompressibilityHint { get; set; } = 0.35;
public string? ReceiptsDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";

public ulong BlocksDbWriteBufferSize { get; set; } = (ulong)64.MiB();
public uint BlocksDbWriteBufferNumber { get; set; } = 2;
Expand All @@ -78,7 +80,7 @@ public class DbConfig : IDbConfig
public bool? BlocksDbUseDirectReads { get; set; }
public bool? BlocksDbUseDirectIoForFlushAndCompactions { get; set; }
public ulong? BlocksDbCompactionReadAhead { get; set; }
public string? BlocksDbAdditionalRocksDbOptions { get; set; }
public string? BlocksDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";

public ulong HeadersDbWriteBufferSize { get; set; } = (ulong)8.MiB();
public uint HeadersDbWriteBufferNumber { get; set; } = 2;
Expand All @@ -90,7 +92,7 @@ public class DbConfig : IDbConfig
public bool? HeadersDbUseDirectReads { get; set; }
public bool? HeadersDbUseDirectIoForFlushAndCompactions { get; set; }
public ulong? HeadersDbCompactionReadAhead { get; set; }
public string? HeadersDbAdditionalRocksDbOptions { get; set; }
public string? HeadersDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";
public ulong? HeadersDbMaxBytesForLevelBase { get; set; } = (ulong)128.MiB();

public ulong BlockNumbersDbWriteBufferSize { get; set; } = (ulong)8.MiB();
Expand Down Expand Up @@ -119,7 +121,7 @@ public class DbConfig : IDbConfig
public bool? BlockInfosDbUseDirectReads { get; set; }
public bool? BlockInfosDbUseDirectIoForFlushAndCompactions { get; set; }
public ulong? BlockInfosDbCompactionReadAhead { get; set; }
public string? BlockInfosDbAdditionalRocksDbOptions { get; set; }
public string? BlockInfosDbAdditionalRocksDbOptions { get; set; } = "compaction_pri=kOldestLargestSeqFirst";

public ulong PendingTxsDbWriteBufferSize { get; set; } = (ulong)4.MiB();
public uint PendingTxsDbWriteBufferNumber { get; set; } = 4;
Expand Down Expand Up @@ -202,6 +204,7 @@ public class DbConfig : IDbConfig
public int? StateDbUseRibbonFilterStartingFromLevel { get; set; } = 2;
public double? StateDbDataBlockIndexUtilRatio { get; set; } = 0.5;
public bool StateDbEnableFileWarmer { get; set; } = false;
public double StateDbCompressibilityHint { get; set; } = 0.45;
public string? StateDbAdditionalRocksDbOptions { get; set; }

public uint RecycleLogFileNum { get; set; } = 0;
Expand Down
3 changes: 3 additions & 0 deletions src/Nethermind/Nethermind.Db.Rocks/Config/IDbConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public interface IDbConfig : IConfig
ulong BytesPerSync { get; set; }
double? DataBlockIndexUtilRatio { get; set; }
bool EnableFileWarmer { get; set; }
double CompressibilityHint { get; set; }

ulong BlobTransactionsDbBlockCacheSize { get; set; }

Expand All @@ -67,6 +68,7 @@ public interface IDbConfig : IConfig
bool? ReceiptsDbUseDirectIoForFlushAndCompactions { get; set; }
ulong? ReceiptsDbCompactionReadAhead { get; set; }
ulong ReceiptsDbTargetFileSizeBase { get; set; }
double ReceiptsDbCompressibilityHint { get; set; }
string? ReceiptsDbAdditionalRocksDbOptions { get; set; }

ulong BlocksDbWriteBufferSize { get; set; }
Expand Down Expand Up @@ -203,6 +205,7 @@ public interface IDbConfig : IConfig
int? StateDbUseRibbonFilterStartingFromLevel { get; set; }
double? StateDbDataBlockIndexUtilRatio { get; set; }
bool StateDbEnableFileWarmer { get; set; }
double StateDbCompressibilityHint { get; set; }
string? StateDbAdditionalRocksDbOptions { get; set; }

/// <summary>
Expand Down
70 changes: 47 additions & 23 deletions src/Nethermind/Nethermind.Db.Rocks/Config/PerTableDbConfig.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
// SPDX-License-Identifier: LGPL-3.0-only

using System;
using System.Collections.Generic;
using System.IO;
using System.Reflection;
using Nethermind.Core.Extensions;
Expand All @@ -12,6 +11,7 @@ namespace Nethermind.Db.Rocks.Config;
public class PerTableDbConfig
{
private readonly string _tableName;
private readonly string? _columnName;
private readonly IDbConfig _dbConfig;
private readonly DbSettings _settings;

Expand All @@ -20,10 +20,7 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum
_dbConfig = dbConfig;
_settings = dbSettings;
_tableName = _settings.DbName;
if (columnName is not null)
{
_tableName += columnName;
}
_columnName = columnName;
}

public bool CacheIndexAndFilterBlocks => _settings.CacheIndexAndFilterBlocks ?? ReadConfig<bool>(nameof(CacheIndexAndFilterBlocks));
Expand Down Expand Up @@ -74,46 +71,73 @@ public PerTableDbConfig(IDbConfig dbConfig, DbSettings dbSettings, string? colum
public ulong BytesPerSync => ReadConfig<ulong>(nameof(BytesPerSync));
public double? DataBlockIndexUtilRatio => ReadConfig<double?>(nameof(DataBlockIndexUtilRatio));
public bool EnableFileWarmer => ReadConfig<bool>(nameof(EnableFileWarmer));
public double CompressibilityHint => ReadConfig<double>(nameof(CompressibilityHint));

private T? ReadConfig<T>(string propertyName)
{
return ReadConfig<T>(_dbConfig, propertyName, GetPrefix());
return ReadConfig<T>(_dbConfig, propertyName, GetPrefixes());
}

private string GetPrefix()
private string[] GetPrefixes()
{
return _tableName.StartsWith("State") ? "StateDb" : string.Concat(_tableName, "Db");
if (_tableName.StartsWith("State"))
{
return ["StateDb"];
}

if (_columnName != null)
{
return [
string.Concat(_tableName, _columnName, "Db"),
string.Concat(_tableName, "Db"),
];
}

return [string.Concat(_tableName, "Db")];
}

private static T? ReadConfig<T>(IDbConfig dbConfig, string propertyName, string prefix)
private static T? ReadConfig<T>(IDbConfig dbConfig, string propertyName, string[] prefixes)
{
string prefixed = string.Concat(prefix, propertyName);

try
{
Type type = dbConfig.GetType();
PropertyInfo? propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
PropertyInfo? propertyInfo;

if (propertyInfo is not null && propertyInfo.PropertyType.CanBeAssignedNull())
foreach (var prefix in prefixes)
{
// If its nullable check if its null first
T? val = (T?)propertyInfo?.GetValue(dbConfig);
if (val is not null)
string prefixed = string.Concat(prefix, propertyName);

propertyInfo = type.GetProperty(prefixed, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
if (propertyInfo is not null)
{
return val;
if (propertyInfo.PropertyType.CanBeAssignedNull())
{
// If its nullable check if its null first
object? valObj = propertyInfo.GetValue(dbConfig);
if (valObj is not null)
{
T? val = (T?)valObj;
if (val is not null)
{
return val;
}
}
}
else
{
// If not nullable just use it directly
return (T?)propertyInfo.GetValue(dbConfig);
}
}

// Use generic one even if its available
propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
}

// if no custom db property default to generic one
propertyInfo ??= type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
// Use generic one even if its available
propertyInfo = type.GetProperty(propertyName, BindingFlags.IgnoreCase | BindingFlags.Public | BindingFlags.Instance);
return (T?)propertyInfo?.GetValue(dbConfig);
}
catch (Exception e)
{
throw new InvalidDataException($"Unable to read {prefixed} property from DB config", e);
throw new InvalidDataException($"Unable to read property from DB config. Prefixes: ${prefixes}", e);
}
}
}
8 changes: 5 additions & 3 deletions src/Nethermind/Nethermind.Db.Rocks/DbOnTheRocks.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1500,7 +1500,7 @@ public virtual void Tune(ITunableDb.TuneType type)
case ITunableDb.TuneType.HeavyWrite:
// Compaction spikes are clear at this point. Will definitely affect attestation performance.
// Its unclear if it improve or slow down sync time. Seems to be the sweet spot.
ApplyOptions(GetHeavyWriteOptions((ulong)4.GiB()));
ApplyOptions(GetHeavyWriteOptions((ulong)2.GiB()));
break;
case ITunableDb.TuneType.AggressiveHeavyWrite:
// For when, you are desperate, but don't wanna disable compaction completely, because you don't want
Expand Down Expand Up @@ -1609,8 +1609,10 @@ private IDictionary<string, string> GetHeavyWriteOptions(ulong l0SizeTarget)
// but no io, only cpu.
// bufferSize*maxBufferNumber = 128MB, which is the max memory used, which tend to be the case as its now
// stalled by compaction instead of flush.
ulong bufferSize = (ulong)16.MiB();
ulong l0FileSize = bufferSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge;
// The buffer is not compressed unlike l0File, so to account for it, its size need to be slightly larger.
ulong targetFileSize = (ulong)16.MiB();
ulong bufferSize = (ulong)(targetFileSize / _perTableDbConfig.CompressibilityHint);
ulong l0FileSize = targetFileSize * (ulong)_perTableDbConfig.MinWriteBufferNumberToMerge;
ulong maxBufferNumber = 8;

// Guide recommend to have l0 and l1 to be the same size. They have to be compacted together so if l1 is larger,
Expand Down
11 changes: 11 additions & 0 deletions src/Nethermind/Nethermind.Db.Test/Config/PerTableDbConfigTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,17 @@ public void CanReadAllConfigForAllTable()
}
}

[Test]
public void When_ColumnDb_UsePerTableConfig()
{
DbConfig dbConfig = new DbConfig();
dbConfig.MaxOpenFiles = 2;
dbConfig.ReceiptsDbMaxOpenFiles = 3;

PerTableDbConfig config = new PerTableDbConfig(dbConfig, new DbSettings(DbNames.Receipts, ""), "Blocks");
config.MaxOpenFiles.Should().Be(3);
}

[Test]
public void When_PerTableConfigIsAvailable_UsePerTableConfig()
{
Expand Down

0 comments on commit 0d700b3

Please sign in to comment.