Integrate EngineCapabilities into HealthChecks

src/Nethermind/Nethermind.Api/IApiWithBlockchain.cs

@@ -17,6 +17,7 @@
 using Nethermind.Evm.TransactionProcessing;
 using Nethermind.Facade;
 using Nethermind.Facade.Eth;
+using Nethermind.JsonRpc;
 using Nethermind.JsonRpc.Modules.Eth.GasPrice;
 using Nethermind.State;
 using Nethermind.Trie.Pruning;
@@ -70,6 +71,7 @@ public interface IApiWithBlockchain : IApiWithStores, IBlockchainBridgeFactory
         IWitnessCollector? WitnessCollector { get; set; }
         IWitnessRepository? WitnessRepository { get; set; }
         IHealthHintService? HealthHintService { get; set; }
+        IRpcCapabilitiesProvider? RpcCapabilitiesProvider { get; set; }
         ITransactionComparerProvider? TransactionComparerProvider { get; set; }
         TxValidator? TxValidator { get; set; }
 

src/Nethermind/Nethermind.Api/NethermindApi.cs

@@ -200,6 +200,7 @@ public ISealEngine SealEngine
         public ITxPool? TxPool { get; set; }
         public ITxPoolInfoProvider? TxPoolInfoProvider { get; set; }
         public IHealthHintService? HealthHintService { get; set; }
+        public IRpcCapabilitiesProvider? RpcCapabilitiesProvider { get; set; }
         public TxValidator? TxValidator { get; set; }
         public IBlockFinalizationManager? FinalizationManager { get; set; }
         public IGasLimitCalculator GasLimitCalculator { get; set; }

src/Nethermind/Nethermind.HealthChecks.Test/NodeHealthServiceTests.cs

@@ -65,7 +65,7 @@ public void CheckHealth_returns_expected_results([ValueSource(nameof(CheckHealth
             IEthSyncingInfo ethSyncingInfo = new EthSyncingInfo(blockFinder, receiptStorage, syncConfig, LimboLogs.Instance);
             NodeHealthService nodeHealthService =
                 new(syncServer, blockchainProcessor, blockProducer, new HealthChecksConfig(),
-                    healthHintService, ethSyncingInfo, api, new[] { drive }, test.IsMining);
+                    healthHintService, ethSyncingInfo, new EngineRpcCapabilitiesProvider(api.SpecProvider), api, new[] { drive }, test.IsMining);
             CheckHealthResult result = nodeHealthService.CheckHealth();
             Assert.AreEqual(test.ExpectedHealthy, result.Healthy);
             Assert.AreEqual(test.ExpectedMessage, FormatMessages(result.Messages.Select(x => x.Message)));
@@ -75,6 +75,9 @@ public void CheckHealth_returns_expected_results([ValueSource(nameof(CheckHealth
         [Test]
         public void post_merge_health_checks([ValueSource(nameof(CheckHealthPostMergeTestCases))] CheckHealthPostMergeTest test)
         {
+            Assert.AreEqual(test.EnabledCapabilities.Length, test.EnabledCapabilitiesUpdatedCalls.Length);
+            Assert.AreEqual(test.DisabledCapabilities.Length, test.DisabledCapabilitiesUpdatedCalls.Length);
+
             IBlockTree blockFinder = Substitute.For<IBlockTree>();
             ISyncServer syncServer = Substitute.For<ISyncServer>();
             IBlockchainProcessor blockchainProcessor = Substitute.For<IBlockchainProcessor>();
@@ -85,11 +88,21 @@ public void post_merge_health_checks([ValueSource(nameof(CheckHealthPostMergeTes
             ManualTimestamper timestamper = new(DateTime.Parse("18:23:00"));
             api.Timestamper.Returns(timestamper);
             api.JsonRpcLocalStats = Substitute.For<IJsonRpcLocalStats>();
-            MethodStats methodStats = new();
-            methodStats.Successes = 0;
-            api.JsonRpcLocalStats!.GetMethodStats("engine_forkchoiceUpdatedV1").Returns(methodStats);
-            api.JsonRpcLocalStats!.GetMethodStats("engine_newPayloadV1").Returns(methodStats);
-            api.JsonRpcLocalStats!.GetMethodStats("engine_exchangeTransitionConfigurationV1").Returns(methodStats);
+
+            MethodStats[] enabledMethodStats = new MethodStats[test.EnabledCapabilities.Length];
+            for (int i = 0; i < enabledMethodStats.Length; i++)
+            {
+                enabledMethodStats[i] = new MethodStats();
+                api.JsonRpcLocalStats!.GetMethodStats(test.EnabledCapabilities[i]).Returns(enabledMethodStats[i]);
+            }
+
+            MethodStats[] disabledMethodStats = new MethodStats[test.DisabledCapabilities.Length];
+            for (int i = 0; i < disabledMethodStats.Length; i++)
+            {
+                disabledMethodStats[i] = new MethodStats();
+                api.JsonRpcLocalStats!.GetMethodStats(test.DisabledCapabilities[i]).Returns(disabledMethodStats[i]);
+            }
+
             syncServer.GetPeerCount().Returns(test.PeerCount);
             IDriveInfo drive = Substitute.For<IDriveInfo>();
             drive.AvailableFreeSpace.Returns(_freeSpaceBytes);
@@ -111,14 +124,24 @@ public void post_merge_health_checks([ValueSource(nameof(CheckHealthPostMergeTes
                 blockFinder.FindBestSuggestedHeader().Returns(GetBlockHeader(2).TestObject);
             }
 
+            CustomRpcCapabilitiesProvider customProvider =
+                new(test.EnabledCapabilities, test.DisabledCapabilities);
             IEthSyncingInfo ethSyncingInfo = new EthSyncingInfo(blockFinder, new InMemoryReceiptStorage(), new SyncConfig(), new TestLogManager());
             NodeHealthService nodeHealthService =
                 new(syncServer, blockchainProcessor, blockProducer, new HealthChecksConfig(),
-                    healthHintService, ethSyncingInfo, api, new[] { drive }, false);
+                    healthHintService, ethSyncingInfo, customProvider, api, new[] { drive }, false);
             nodeHealthService.CheckHealth();
 
             timestamper.Add(TimeSpan.FromSeconds(test.TimeSpanSeconds));
-            methodStats.Successes = test.ForkchoiceUpdatedCalls;
+            for (int i = 0; i < enabledMethodStats.Length; i++)
+            {
+                enabledMethodStats[i].Successes = test.EnabledCapabilitiesUpdatedCalls[i];
+            }
+
+            for (int i = 0; i < disabledMethodStats.Length; i++)
+            {
+                disabledMethodStats[i].Successes = test.DisabledCapabilitiesUpdatedCalls[i];
+            }
 
             CheckHealthResult result = nodeHealthService.CheckHealth();
             Assert.AreEqual(test.ExpectedHealthy, result.Healthy);
@@ -139,7 +162,13 @@ public class CheckHealthPostMergeTest
 
             public string ExpectedLongMessage { get; set; }
 
-            public int ForkchoiceUpdatedCalls { get; set; }
+            public int[] EnabledCapabilitiesUpdatedCalls { get; set; }
+
+            public int[] DisabledCapabilitiesUpdatedCalls { get; set; } = Array.Empty<int>();
+
+            public string[] EnabledCapabilities { get; set; }
+
+            public string[] DisabledCapabilities { get; set; } = Array.Empty<string>();
 
             public int TimeSpanSeconds { get; set; }
             public double AvailableDiskSpacePercent { get; set; } = 11;
@@ -291,7 +320,8 @@ public static IEnumerable<CheckHealthPostMergeTest> CheckHealthPostMergeTestCase
                     ExpectedHealthy = false,
                     ExpectedMessage = "Fully synced. Peers: 10. No messages from CL.",
                     TimeSpanSeconds = 301,
-                    ForkchoiceUpdatedCalls = 0,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 0, 0, 0 },
                     ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10. No new messages from CL after last check."
                 };
                 yield return new CheckHealthPostMergeTest()
@@ -302,7 +332,8 @@ public static IEnumerable<CheckHealthPostMergeTest> CheckHealthPostMergeTestCase
                     ExpectedHealthy = true,
                     ExpectedMessage = "Fully synced. Peers: 10.",
                     TimeSpanSeconds = 15,
-                    ForkchoiceUpdatedCalls = 0,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 0, 0, 0 },
                     ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10."
                 };
                 yield return new CheckHealthPostMergeTest()
@@ -313,7 +344,8 @@ public static IEnumerable<CheckHealthPostMergeTest> CheckHealthPostMergeTestCase
                     ExpectedHealthy = true,
                     ExpectedMessage = "Fully synced. Peers: 10.",
                     TimeSpanSeconds = 301,
-                    ForkchoiceUpdatedCalls = 1,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 1, 1, 1 },
                     ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10."
                 };
                 yield return new CheckHealthPostMergeTest()
@@ -324,7 +356,36 @@ public static IEnumerable<CheckHealthPostMergeTest> CheckHealthPostMergeTestCase
                     ExpectedHealthy = true,
                     ExpectedMessage = "Fully synced. Peers: 10.",
                     TimeSpanSeconds = 15,
-                    ForkchoiceUpdatedCalls = 1,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 1, 1, 1 },
+                    ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10."
+                };
+                yield return new CheckHealthPostMergeTest()
+                {
+                    Lp = 4,
+                    IsSyncing = false,
+                    PeerCount = 10,
+                    ExpectedHealthy = false,
+                    ExpectedMessage = "Fully synced. Peers: 10. No messages from CL.",
+                    TimeSpanSeconds = 301,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 0, 0, 0 },
+                    DisabledCapabilities = new[] { "X", "Y", "Z" },
+                    DisabledCapabilitiesUpdatedCalls = new[] { 1, 1, 1 },
+                    ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10. No new messages from CL after last check."
+                };
+                yield return new CheckHealthPostMergeTest()
+                {
+                    Lp = 4,
+                    IsSyncing = false,
+                    PeerCount = 10,
+                    ExpectedHealthy = true,
+                    ExpectedMessage = "Fully synced. Peers: 10.",
+                    TimeSpanSeconds = 301,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 0, 1, 0 },
+                    DisabledCapabilities = new[] { "X", "Y", "Z" },
+                    DisabledCapabilitiesUpdatedCalls = new[] { 0, 0, 0 },
                     ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10."
                 };
                 yield return new CheckHealthPostMergeTest()
@@ -334,19 +395,33 @@ public static IEnumerable<CheckHealthPostMergeTest> CheckHealthPostMergeTestCase
                     PeerCount = 10,
                     ExpectedHealthy = false,
                     ExpectedMessage = "Still syncing. Peers: 10.",
-                    TimeSpanSeconds = 15,
-                    ForkchoiceUpdatedCalls = 1,
+                    TimeSpanSeconds = 301,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 1, 1, 1 },
                     ExpectedLongMessage = "The node is still syncing, CurrentBlock: 4, HighestBlock: 15. The status will change to healthy once synced. Peers: 10."
                 };
                 yield return new CheckHealthPostMergeTest()
+                {
+                    Lp = 5,
+                    IsSyncing = false,
+                    PeerCount = 10,
+                    ExpectedHealthy = true,
+                    ExpectedMessage = "Fully synced. Peers: 10.",
+                    TimeSpanSeconds = 301,
+                    EnabledCapabilities = new[] { "engine_forkchoiceUpdatedV999", "engine_newPayloadV999" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 1, 1 },
+                    ExpectedLongMessage = "The node is now fully synced with a network. Peers: 10."
+                };
+                yield return new CheckHealthPostMergeTest()
                 {
                     Lp = 6,
                     IsSyncing = false,
                     PeerCount = 10,
                     ExpectedHealthy = false,
                     ExpectedMessage = "Fully synced. Peers: 10. Low free disk space.",
                     TimeSpanSeconds = 15,
-                    ForkchoiceUpdatedCalls = 1,
+                    EnabledCapabilities = new[] { "A", "B", "C" },
+                    EnabledCapabilitiesUpdatedCalls = new[] { 1, 1, 1 },
                     AvailableDiskSpacePercent = 4.73,
                     ExpectedLongMessage = $"The node is now fully synced with a network. Peers: 10. The node is running out of free disk space in 'C:/' - only {1.50:F2} GB ({4.73:F2}%) left."
                 };
@@ -366,5 +441,28 @@ private static string FormatMessages(IEnumerable<string> messages)
 
             return string.Empty;
         }
+
+        private class CustomRpcCapabilitiesProvider : IRpcCapabilitiesProvider
+        {
+            private readonly Dictionary<string, bool> _capabilities = new();
+
+            public CustomRpcCapabilitiesProvider(IReadOnlyList<string> enabledCapabilities, IReadOnlyList<string> disabledCapabilities)
+            {
+                foreach (string capability in enabledCapabilities)
+                {
+                    _capabilities[capability] = true;
+                }
+
+                foreach (string capability in disabledCapabilities)
+                {
+                    _capabilities[capability] = false;
+                }
+            }
+
+            public IReadOnlyDictionary<string, bool> GetEngineCapabilities()
+            {
+                return _capabilities;
+            }
+        }
     }
 }

src/Nethermind/Nethermind.HealthChecks/HealthChecksPlugin.cs

@@ -128,7 +128,7 @@ public Task InitRpcModules()
 
             _nodeHealthService = new NodeHealthService(_api.SyncServer,
                 _api.BlockchainProcessor!, _api.BlockProducer!, _healthChecksConfig, _api.HealthHintService!,
-                _api.EthSyncingInfo!, _api, drives, _initConfig.IsMining);
+                _api.EthSyncingInfo!, _api.RpcCapabilitiesProvider, _api, drives, _initConfig.IsMining);
 
             if (_healthChecksConfig.Enabled)
             {

src/Nethermind/Nethermind.HealthChecks/NodeHealthService.cs

@@ -12,6 +12,7 @@
 using Nethermind.Consensus.Processing;
 using Nethermind.Core.Extensions;
 using Nethermind.Facade.Eth;
+using Nethermind.JsonRpc;
 using Nethermind.Synchronization;
 
 namespace Nethermind.HealthChecks
@@ -29,10 +30,10 @@ public class NodeHealthService : INodeHealthService
         private readonly IBlockchainProcessor _blockchainProcessor;
         private readonly IBlockProducer _blockProducer;
         private readonly IHealthChecksConfig _healthChecksConfig;
-        private readonly IInitConfig _initConfig;
         private readonly IHealthHintService _healthHintService;
         private readonly IEthSyncingInfo _ethSyncingInfo;
         private readonly INethermindApi _api;
+        private readonly IRpcCapabilitiesProvider _rpcCapabilitiesProvider;
         private readonly IDriveInfo[] _drives;
         private readonly bool _isMining;
 
@@ -42,18 +43,19 @@ public NodeHealthService(ISyncServer syncServer,
             IHealthChecksConfig healthChecksConfig,
             IHealthHintService healthHintService,
             IEthSyncingInfo ethSyncingInfo,
+            IRpcCapabilitiesProvider rpcCapabilitiesProvider,
             INethermindApi api,
             IDriveInfo[] drives,
             bool isMining)
         {
             _syncServer = syncServer;
             _isMining = isMining;
             _healthChecksConfig = healthChecksConfig;
-            _initConfig = api.Config<IInitConfig>();
             _healthHintService = healthHintService;
             _blockchainProcessor = blockchainProcessor;
             _blockProducer = blockProducer;
             _ethSyncingInfo = ethSyncingInfo;
+            _rpcCapabilitiesProvider = rpcCapabilitiesProvider;
             _api = api;
             _drives = drives;
         }
@@ -144,10 +146,15 @@ public CheckHealthResult CheckHealth()
         public bool CheckClAlive()
         {
             var now = _api.Timestamper.UtcNow;
-            bool forkchoice = CheckMethodInvoked("engine_forkchoiceUpdatedV1", now);
-            bool newPayload = CheckMethodInvoked("engine_newPayloadV1", now);
-            bool exchangeTransition = CheckMethodInvoked("engine_exchangeTransitionConfigurationV1", now);
-            return forkchoice || newPayload || exchangeTransition;
+            bool result = false;
+            foreach (var capability in _rpcCapabilitiesProvider.GetEngineCapabilities())
+            {
+                if (capability.Value)
+                {
+                    result |= CheckMethodInvoked(capability.Key, now);
+                }
+            }
+            return result;
         }
 
         private readonly ConcurrentDictionary<string, bool> _previousMethodCheckResult = new();

src/Nethermind/Nethermind.JsonRpc/IRpcCapabilitiesProvider.cs

@@ -0,0 +1,11 @@
+// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited
+// SPDX-License-Identifier: LGPL-3.0-only
+
+using System.Collections.Generic;
+
+namespace Nethermind.JsonRpc;
+
+public interface IRpcCapabilitiesProvider
+{
+    IReadOnlyDictionary<string, bool> GetEngineCapabilities();
+}

src/Nethermind/Nethermind.Merge.Plugin.Test/EngineModuleTests.Setup.cs

@@ -23,7 +23,9 @@
 using Nethermind.Db;
 using Nethermind.Evm.Tracing;
 using Nethermind.Facade.Eth;
+using Nethermind.HealthChecks;
 using Nethermind.Int256;
+using Nethermind.JsonRpc;
 using Nethermind.Logging;
 using Nethermind.Merge.Plugin.BlockProduction;
 using Nethermind.Merge.Plugin.Handlers;
@@ -62,6 +64,7 @@ private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConf
             chain.LogManager);
         invalidChainTracker.SetupBlockchainProcessorInterceptor(chain.BlockchainProcessor);
         chain.BeaconSync = new BeaconSync(chain.BeaconPivot, chain.BlockTree, syncConfig ?? new SyncConfig(), blockCacheService, chain.LogManager);
+        EngineRpcCapabilitiesProvider capabilitiesProvider = new(chain.SpecProvider);
         return new EngineRpcModule(
             new GetPayloadV1Handler(
                 chain.PayloadPreparationService!,
@@ -99,7 +102,7 @@ private IEngineRpcModule CreateEngineModule(MergeTestBlockchain chain, ISyncConf
             new GetPayloadBodiesByHashV1Handler(chain.BlockTree, chain.LogManager),
             new GetPayloadBodiesByRangeV1Handler(chain.BlockTree, chain.LogManager),
             new ExchangeTransitionConfigurationV1Handler(chain.PoSSwitcher, chain.LogManager),
-            new ExchangeCapabilitiesHandler(chain.SpecProvider, chain.LogManager),
+            new ExchangeCapabilitiesHandler(capabilitiesProvider, chain.SpecProvider, chain.LogManager),
             chain.SpecProvider,
             chain.LogManager);
     }