Kiln restarts + fixes

src/Nethermind/Nethermind.Consensus/Processing/BlockchainProcessor.cs

@@ -504,7 +504,6 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
             List<Block> blocksToBeAddedToMain = new();
 
             bool preMergeFinishBranchingCondition;
-            bool postMergeFinishBranchingCondition;
             bool suggestedBlockIsPostMerge = suggestedBlock.IsPostMerge;
 
             Block toBeProcessed = suggestedBlock;
@@ -568,15 +567,11 @@ private ProcessingBranch PrepareProcessingBranch(Block suggestedBlock, Processin
                 // otherwise some nodes would be missing
                 bool notFoundTheBranchingPointYet = !_blockTree.IsMainChain(branchingPoint.Hash!);
                 bool notReachedTheReorgBoundary = branchingPoint.Number > (_blockTree.Head?.Header.Number ?? 0);
-                preMergeFinishBranchingCondition = (notFoundTheBranchingPointYet || notReachedTheReorgBoundary) &&
-                                                   !suggestedBlockIsPostMerge;
-                postMergeFinishBranchingCondition = suggestedBlockIsPostMerge &&
-                                                    _blockTree.WasProcessed(branchingPoint.Number,
-                                                        branchingPoint.Hash) == false;
+                preMergeFinishBranchingCondition = (notFoundTheBranchingPointYet || notReachedTheReorgBoundary);
                 if (_logger.IsTrace)
                     _logger.Trace(
-                        $" Current branching point: {branchingPoint.Number}, {branchingPoint.Hash} TD: {branchingPoint.TotalDifficulty} Processing conditions notFoundTheBranchingPointYet {notFoundTheBranchingPointYet}, notReachedTheReorgBoundary: {notReachedTheReorgBoundary}, suggestedBlockIsPostMerge {suggestedBlockIsPostMerge}, postMergeFinishBranchingCondition: {postMergeFinishBranchingCondition}");
-            } while (preMergeFinishBranchingCondition || postMergeFinishBranchingCondition);
+                        $" Current branching point: {branchingPoint.Number}, {branchingPoint.Hash} TD: {branchingPoint.TotalDifficulty} Processing conditions notFoundTheBranchingPointYet {notFoundTheBranchingPointYet}, notReachedTheReorgBoundary: {notReachedTheReorgBoundary}, suggestedBlockIsPostMerge {suggestedBlockIsPostMerge}");
+            } while (preMergeFinishBranchingCondition);
 
             if (branchingPoint != null && branchingPoint.Hash != _blockTree.Head?.Hash)
             {

src/Nethermind/Nethermind.Core/ChainLevelInfo.cs

@@ -36,22 +36,26 @@ public ChainLevelInfo(bool hasBlockInMainChain, params BlockInfo[] blockInfos)
         public BlockInfo[] BlockInfos { get; set; }
         public BlockInfo? MainChainBlock => HasBlockOnMainChain ? BlockInfos[0] : null;
 
+        // ToDo we need to rethink this code
         public BlockInfo? BeaconMainChainBlock
         {
             get
             {
                 if (MainChainBlock != null)
                     return MainChainBlock;
 
+                if (BlockInfos.Length == 0)
+                    return null;
+
                 for (int i = 0; i < BlockInfos.Length; ++i)
                 {
                     BlockInfo blockInfo = BlockInfos[i];
                     bool isBeaconChainMetadata = (blockInfo.Metadata & BlockMetadata.BeaconMainChain) != 0;
                     if (isBeaconChainMetadata)
                         return blockInfo;
                 }
-
-                return null;
+                
+                return BlockInfos[0];
             }
         }
     }

src/Nethermind/Nethermind.Merge.Plugin.Test/MergePluginTests.cs

@@ -50,6 +50,7 @@ public void Setup()
             _context.ConfigProvider.GetConfig<IMergeConfig>().Returns(_mergeConfig);
             _context.ConfigProvider.GetConfig<ISyncConfig>().Returns(new SyncConfig());
             _context.ConfigProvider.GetConfig<IMiningConfig>().Returns(miningConfig);
+            _context.BlockProcessingQueue?.IsEmpty.Returns(true);
             _context.MemDbFactory = new MemDbFactory();
             _context.BlockProducerEnvFactory = new BlockProducerEnvFactory(
                 _context.DbProvider!,

src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs

@@ -185,33 +185,35 @@ public async Task<ResultWrapper<ForkchoiceUpdatedV1Result>> Handle(ForkchoiceSta
             {
                 _blockTree.UpdateMainChain(blocks!, true, true);
             }
-
+            
             bool nonZeroFinalizedBlockHash = forkchoiceState.FinalizedBlockHash != Keccak.Zero;
             bool nonZeroSafeBlockHash = forkchoiceState.SafeBlockHash != Keccak.Zero;
-            bool finalizedBlockHashInconsistent = nonZeroFinalizedBlockHash && !_blockTree.IsMainChain(finalizedHeader!);
-            if (finalizedBlockHashInconsistent)
-            {
-                string errorMsg = $"Inconsistent forkchoiceState - finalized block hash. Request: {requestStr}";
-                if (_logger.IsWarn)
-                    _logger.Warn(errorMsg);
 
-                return ForkchoiceUpdatedV1Result.Error(errorMsg, MergeErrorCodes.InvalidForkchoiceState);
-            }
-
-            bool safeBlockHashInconsistent = nonZeroSafeBlockHash && !_blockTree.IsMainChain(safeBlockHashHeader!);
-            if (safeBlockHashInconsistent)
-            {
-                string errorMsg = $"Inconsistent forkchoiceState - safe block hash. Request: {requestStr}";
-                if (_logger.IsWarn)
-                    _logger.Warn(errorMsg);
-
-                return ForkchoiceUpdatedV1Result.Error(errorMsg, MergeErrorCodes.InvalidForkchoiceState);
-            }
-
-            if (nonZeroFinalizedBlockHash)
-            {
-                _manualBlockFinalizationManager.MarkFinalized(newHeadBlock!.Header, finalizedHeader!);
-            }
+            /*This checks will be uncommented in next release. We need to check hive tests*/
+            // bool finalizedBlockHashInconsistent = nonZeroFinalizedBlockHash && !_blockTree.IsMainChain(finalizedHeader!);
+            // if (finalizedBlockHashInconsistent)
+            // {
+            //     string errorMsg = $"Inconsistent forkchoiceState - finalized block hash. Request: {requestStr}";
+            //     if (_logger.IsWarn)
+            //         _logger.Warn(errorMsg);
+            //
+            //     return ForkchoiceUpdatedV1Result.Error(errorMsg, MergeErrorCodes.InvalidForkchoiceState);
+            // }
+            //
+            // bool safeBlockHashInconsistent = nonZeroSafeBlockHash && !_blockTree.IsMainChain(safeBlockHashHeader!);
+            // if (safeBlockHashInconsistent)
+            // {
+            //     string errorMsg = $"Inconsistent forkchoiceState - safe block hash. Request: {requestStr}";
+            //     if (_logger.IsWarn)
+            //         _logger.Warn(errorMsg);
+            //
+            //     return ForkchoiceUpdatedV1Result.Error(errorMsg, MergeErrorCodes.InvalidForkchoiceState);
+            // }
+            //
+            // if (nonZeroFinalizedBlockHash)
+            // {
+            //     _manualBlockFinalizationManager.MarkFinalized(newHeadBlock!.Header, finalizedHeader!);
+            // }
 
             // In future safeBlockHash will be added to JSON-RPC
             if (nonZeroSafeBlockHash)

src/Nethermind/Nethermind.Merge.Plugin/MergePlugin.cs

@@ -15,12 +15,14 @@
 //  along with the Nethermind. If not, see <http://www.gnu.org/licenses/>.
 
 using System;
+using System.Threading;
 using System.Threading.Tasks;
 using Nethermind.Api;
 using Nethermind.Api.Extensions;
 using Nethermind.Blockchain;
 using Nethermind.Blockchain.Synchronization;
 using Nethermind.Consensus;
+using Nethermind.Consensus.Processing;
 using Nethermind.Consensus.Rewards;
 using Nethermind.Consensus.Validators;
 using Nethermind.Core;
@@ -74,7 +76,7 @@ public Task Init(INethermindApi nethermindApi)
                 if (_api.SealValidator == null) throw new ArgumentException(nameof(_api.SealValidator));
 
                 _blockCacheService = new BlockCacheService();
-                _poSSwitcher = new PoSSwitcher(_mergeConfig, new SyncConfig(),
+                _poSSwitcher = new PoSSwitcher(_mergeConfig, _syncConfig,
                     _api.DbProvider.GetDb<IDb>(DbNames.Metadata), _api.BlockTree, _api.SpecProvider, _blockCacheService, _api.LogManager);
                 _blockFinalizationManager = new ManualBlockFinalizationManager();
 
@@ -141,6 +143,13 @@ public Task InitRpcModules()
                 if (_postMergeBlockProducer is null) throw new ArgumentNullException(nameof(_postMergeBlockProducer));
                 if (_blockProductionTrigger is null) throw new ArgumentNullException(nameof(_blockProductionTrigger));
 
+                // ToDo: ugly temporary hack to not receive engine API messages before end of processing of all blocks after restart. Then we will wait 5s more to ensure everything is processed
+                while (!_api.BlockProcessingQueue.IsEmpty)
+                {
+                    Thread.Sleep(100);
+                }
+                Thread.Sleep(5000);
+
                 PayloadPreparationService payloadPreparationService = new (_postMergeBlockProducer, _blockProductionTrigger, _api.Sealer, _mergeConfig, TimerFactory.Default, _api.LogManager);
 
                 IEngineRpcModule engineRpcModule = new EngineRpcModule(

src/Nethermind/Nethermind.Merge.Plugin/PoSSwitcher.cs

@@ -90,14 +90,13 @@ private void Initialize()
         {
             LoadTerminalBlock();
             LoadFinalizedBlockHash();
-            LoadFinalTotalDifficulty();
-            LoadFinalTotalDifficulty();
 
             if (_terminalBlockNumber != null || _finalTotalDifficulty != null)
                 _hasEverReachedTerminalDifficulty = true;
 
             _specProvider.UpdateMergeTransitionInfo(_firstPoSBlockNumber, _mergeConfig.TerminalTotalDifficultyParsed);
-
+
+            LoadFinalTotalDifficulty();
             if (_terminalBlockNumber == null)
                 _blockTree.NewHeadBlock += CheckIfTerminalBlockReached;
 
@@ -111,7 +110,7 @@ private void LoadFinalTotalDifficulty()
             _finalTotalDifficulty = _mergeConfig.FinalTotalDifficultyParsed;
 
             // pivot post TTD, so we know FinalTotalDifficulty
-            if (_syncConfig.PivotTotalDifficultyParsed >= TerminalTotalDifficulty)
+            if (_syncConfig.PivotTotalDifficultyParsed != 0 && TerminalTotalDifficulty !=null && _syncConfig.PivotTotalDifficultyParsed >= TerminalTotalDifficulty)
             {
                 _finalTotalDifficulty = _syncConfig.PivotTotalDifficultyParsed;
             }

src/Nethermind/Nethermind.Merge.Plugin/Synchronization/MergeBlockDownloader.cs

@@ -210,6 +210,8 @@ bool HasMoreToSync()
                     if (HandleAddResult(bestPeer, currentBlock.Header, blockIndex == 0,
                             _blockTree.SuggestBlock(currentBlock, suggestOptions)))
                     {
+                        if (shouldProcess == false)
+                            _blockTree.UpdateMainChain(new[] { currentBlock }, false);
                         TryUpdateTerminalBlock(currentBlock.Header, shouldProcess);
 
                         if (downloadReceipts)

src/Nethermind/Nethermind.Runner/NLog.config

@@ -60,9 +60,9 @@
  <!--    <logger name="Merge.Plugin.PoSSwitcher" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
  <!--    <logger name="Merge.Plugin.PoSSwitcher" final="true"/> -->
 
- <!--    <logger name="Merge.Plugin.Synchronization.*" minlevel="Trace" writeTo="file-async"/> -->
- <!--    <logger name="Merge.Plugin.Synchronization.*" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
- <!--    <logger name="Merge.Plugin.Synchronization.*" final="true"/> -->
+    <!-- <logger name="Merge.Plugin.Synchronization.*" minlevel="Trace" writeTo="file-async"/> -->
+    <!-- <logger name="Merge.Plugin.Synchronization.*" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
+    <!-- <logger name="Merge.Plugin.Synchronization.*" final="true"/> -->
 
     <!-- <logger name="Synchronization.*" minlevel="Trace" writeTo="file-async"/> -->
     <!-- <logger name="Synchronization.*" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
@@ -86,13 +86,13 @@
     <!-- <logger name="Synchronization.Peers.SyncPeersReport" minlevel="Error" writeTo="auto-colored-console-async"/> -->
     <!-- <logger name="Synchronization.Peers.SyncPeersReport" final="true"/> -->
 
-   <!--  <logger name="Blockchain.BlockTree" minlevel="Trace" writeTo="file-async"/> -->
-   <!--  <logger name="Blockchain.BlockTree" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
-   <!--  <logger name="Blockchain.BlockTree" final="true"/> -->
-   <!--   -->
+    <!-- <logger name="Blockchain.BlockTree" minlevel="Trace" writeTo="file-async"/> -->
+    <!-- <logger name="Blockchain.BlockTree" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
+    <!-- <logger name="Blockchain.BlockTree" final="true"/> -->
+
 	  <!-- <logger name="Consensus.Processing.BlockchainProcessor" minlevel="Trace" writeTo="file-async"/> -->
-   <!--  <logger name="Consensus.Processing.BlockchainProcessor" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
-   <!--  <logger name="Consensus.Processing.BlockchainProcessor" final="true"/> -->
+    <!-- <logger name="Consensus.Processing.BlockchainProcessor" minlevel="Trace" writeTo="auto-colored-console-async"/> -->
+    <!-- <logger name="Consensus.Processing.BlockchainProcessor" final="true"/> -->
 
     <!-- if sync get stuck this is the best thing to enable the Trace on -->
     <!-- <logger name="Synchronization.ParallelSync.MultiSyncModeSelector" minlevel="Trace" writeTo="file-async"/> -->