diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.ChainLevelHelper.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.ChainLevelHelper.cs index 875bcc0d0d9..a89c8db2089 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.ChainLevelHelper.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.ChainLevelHelper.cs @@ -62,7 +62,8 @@ public void Correct_levels_after_chain_level_sync() .AssertBestKnownNumber(9) .AssertBestSuggestedHeader(9) .AssertBestSuggestedBody(9, 10000000) - .AssertChainLevel(0, 9); + .AssertChainLevel(0, 9) + .AssertNotForceNewBeaconSync(); } [Test] @@ -109,4 +110,18 @@ public void Correct_levels_with_chain_fork() .AssertBestSuggestedBody(9) .AssertChainLevel(0, 9); } + + [Test] + public void Correct_levels_after_chain_level_sync_with_disconnected_beacon_chain() + { + BlockTreeTestScenario.GoesLikeThis() + .WithBlockTrees(4, 15) + .InsertBeaconPivot(11) + .SetProcessDestination(11) + .InsertBeaconHeaders(4, 6) + .InsertBeaconHeaders(8, 10) + .SuggestBlocksUsingChainLevels(20) + .AssertChainLevel(0, 4) + .AssertForceNewBeaconSync(); + } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs index 25e12f242e2..749c4edb2c7 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs @@ -375,6 +375,12 @@ public ScenarioBuilder InsertBeaconPivot(long num) return this; } + public ScenarioBuilder SetProcessDestination(long num) + { + _beaconPivot.ProcessDestination = SyncedTree.FindHeader(num, BlockTreeLookupOptions.None); + return this; + } + public ScenarioBuilder ClearBeaconPivot() { NotSyncedTreeBuilder.MetadataDb.Delete(MetadataDbKeys.BeaconSyncPivotNumber); @@ -424,7 +430,7 @@ public ScenarioBuilder SuggestBlocksUsingChainLevels(int maxCount = 2, long maxH } AddBlockResult insertResult = NotSyncedTree.SuggestBlock(beaconBlock, BlockTreeSuggestOptions.ShouldProcess | BlockTreeSuggestOptions.FillBeaconBlock | BlockTreeSuggestOptions.ForceSetAsMain); - Assert.True(AddBlockResult.Added == insertResult, $"BeaconBlock {beaconBlock!.ToString(Block.Format.FullHashAndNumber)}"); + Assert.True(AddBlockResult.Added == insertResult, $"BeaconBlock {beaconBlock!.ToString(Block.Format.FullHashAndNumber)} result {insertResult}"); } headers = _chainLevelHelper!.GetNextHeaders(maxCount, maxHeaderNumber, 0); @@ -635,6 +641,20 @@ public ScenarioBuilder AssertChainLevel(int startNumber, int finalNumber) return this; } + public ScenarioBuilder AssertForceNewBeaconSync() + { + _beaconPivot.ShouldForceStartNewSync.Should().BeTrue(); + + return this; + } + + public ScenarioBuilder AssertNotForceNewBeaconSync() + { + _beaconPivot.ShouldForceStartNewSync.Should().BeFalse(); + + return this; + } + public ScenarioBuilder print() { // Console.WriteLine("LowestInsertedBeaconHeader:"+_notSyncedTree!.LowestInsertedBeaconHeader.Number); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs index ff0add95333..66c0440fb3b 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs @@ -126,6 +126,16 @@ public Task> Handle(ForkchoiceStateV1 f return ForkchoiceUpdatedV1Result.Syncing; } + if (_beaconPivot.ShouldForceStartNewSync) + { + if (_logger.IsInfo) + _logger.Info($"Force starting new sync."); + + StartNewBeaconHeaderSync(forkchoiceState, newHeadBlock!, requestStr); + + return ForkchoiceUpdatedV1Result.Syncing; + } + if (!blockInfo.IsBeaconMainChain && blockInfo.IsBeaconInfo) ReorgBeaconChainDuringSync(newHeadBlock!, blockInfo); diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/BeaconPivot.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/BeaconPivot.cs index 8c7b9467075..a5d053647c5 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/BeaconPivot.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/BeaconPivot.cs @@ -73,6 +73,7 @@ public BeaconPivot( public Keccak PivotHash => CurrentBeaconPivot?.Hash ?? _syncConfig.PivotHashParsed; public BlockHeader? ProcessDestination { get; set; } + public bool ShouldForceStartNewSync { get; set; } = false; // We actually start beacon header sync from the pivot parent hash because hive test.... And because // we can I guess? @@ -132,6 +133,7 @@ public void EnsurePivot(BlockHeader? blockHeader, bool updateOnlyIfNull = false) BlockTreeInsertHeaderOptions.BeaconHeaderInsert | BlockTreeInsertHeaderOptions.TotalDifficultyNotNeeded); CurrentBeaconPivot = blockHeader; _blockTree.LowestInsertedBeaconHeader = blockHeader; + ShouldForceStartNewSync = false; if (_logger.IsInfo) _logger.Info($"New beacon pivot: {blockHeader}"); } } @@ -174,5 +176,6 @@ public interface IBeaconPivot : IPivot // as MergeBlockDownloader process higher block, making it somewhat like a lowest processed beacon block. // TODO: Check if we can just re-use pivot and move pivot forward BlockHeader? ProcessDestination { get; set; } + bool ShouldForceStartNewSync { get; set; } } } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/ChainLevelHelper.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/ChainLevelHelper.cs index f283aa217c5..9f064a040c0 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/ChainLevelHelper.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/ChainLevelHelper.cs @@ -52,6 +52,17 @@ public ChainLevelHelper( _logger = logManager.GetClassLogger(); } + private void OnMissingBeaconHeader(long blockNumber) + { + if (_beaconPivot.ProcessDestination?.Number > blockNumber) + { + // For some reason, this block number is missing when it should not. + // anyway, lets just restart the whole sync. + if (_logger.IsWarn) _logger.Warn($"Unable to find beacon header at height {blockNumber}. This is unexpected, forcing a new beacon sync."); + _beaconPivot.ShouldForceStartNewSync = true; + } + } + public BlockHeader[]? GetNextHeaders(int maxCount, long maxHeaderNumber, int skipLastBlockCount = 0) { long? startingPoint = GetStartingPoint(); @@ -74,6 +85,7 @@ public ChainLevelHelper( BlockInfo? beaconMainChainBlock = level?.BeaconMainChainBlock; if (level == null || beaconMainChainBlock == null) { + OnMissingBeaconHeader(startingPoint.Value); if (_logger.IsTrace) _logger.Trace($"ChainLevelHelper.GetNextHeaders - level {startingPoint} not found"); break; @@ -84,6 +96,7 @@ public ChainLevelHelper( if (newHeader == null) { + OnMissingBeaconHeader(startingPoint.Value); if (_logger.IsTrace) _logger.Trace($"ChainLevelHelper - header {startingPoint} not found"); break; } @@ -180,7 +193,11 @@ public bool TrySetNextBlocks(int maxCount, BlockDownloadContext context) if (_logger.IsTrace) _logger.Trace($"ChainLevelHelper. starting point's starting point is {startingPoint}. Best known number: {_blockTree.BestKnownNumber}, Process destination: {_beaconPivot.ProcessDestination?.Number}"); BlockInfo? beaconMainChainBlock = GetBeaconMainChainBlockInfo(startingPoint); - if (beaconMainChainBlock == null) return null; + if (beaconMainChainBlock == null) + { + OnMissingBeaconHeader(startingPoint); + return null; + } if (!beaconMainChainBlock.IsBeaconInfo) { @@ -201,6 +218,7 @@ public bool TrySetNextBlocks(int maxCount, BlockDownloadContext context) BlockInfo? parentBlockInfo = (_blockTree.GetInfo(header.Number - 1, header.ParentHash!)).Info; if (parentBlockInfo == null) { + OnMissingBeaconHeader(header.Number); return null; }