Autorecover when disconnected chain is found (#4816)

* Autorecover when disconnected chain is found * Fix tests
NethermindEth · Oct 26, 2022 · c174ad5 · c174ad5
1 parent 582cb9e
commit c174ad5
Show file tree

Hide file tree

Showing 5 changed files with 69 additions and 3 deletions.
diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.ChainLevelHelper.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.ChainLevelHelper.cs
@@ -62,7 +62,8 @@ public void Correct_levels_after_chain_level_sync()
             .AssertBestKnownNumber(9)
             .AssertBestSuggestedHeader(9)
             .AssertBestSuggestedBody(9, 10000000)
-            .AssertChainLevel(0, 9);
+            .AssertChainLevel(0, 9)
+            .AssertNotForceNewBeaconSync();
     }
 
     [Test]
@@ -109,4 +110,18 @@ public void Correct_levels_with_chain_fork()
             .AssertBestSuggestedBody(9)
             .AssertChainLevel(0, 9);
     }
+
+    [Test]
+    public void Correct_levels_after_chain_level_sync_with_disconnected_beacon_chain()
+    {
+        BlockTreeTestScenario.GoesLikeThis()
+            .WithBlockTrees(4, 15)
+            .InsertBeaconPivot(11)
+            .SetProcessDestination(11)
+            .InsertBeaconHeaders(4, 6)
+            .InsertBeaconHeaders(8, 10)
+            .SuggestBlocksUsingChainLevels(20)
+            .AssertChainLevel(0, 4)
+            .AssertForceNewBeaconSync();
+    }
 }
diff --git a/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs b/src/Nethermind/Nethermind.Merge.Plugin.Test/BlockTreeTests.cs
@@ -375,6 +375,12 @@ public ScenarioBuilder InsertBeaconPivot(long num)
                 return this;
             }
 
+            public ScenarioBuilder SetProcessDestination(long num)
+            {
+                _beaconPivot.ProcessDestination = SyncedTree.FindHeader(num, BlockTreeLookupOptions.None);
+                return this;
+            }
+
             public ScenarioBuilder ClearBeaconPivot()
             {
                 NotSyncedTreeBuilder.MetadataDb.Delete(MetadataDbKeys.BeaconSyncPivotNumber);
@@ -424,7 +430,7 @@ public ScenarioBuilder SuggestBlocksUsingChainLevels(int maxCount = 2, long maxH
                         }
 
                         AddBlockResult insertResult = NotSyncedTree.SuggestBlock(beaconBlock, BlockTreeSuggestOptions.ShouldProcess | BlockTreeSuggestOptions.FillBeaconBlock | BlockTreeSuggestOptions.ForceSetAsMain);
-                        Assert.True(AddBlockResult.Added == insertResult, $"BeaconBlock {beaconBlock!.ToString(Block.Format.FullHashAndNumber)}");
+                        Assert.True(AddBlockResult.Added == insertResult, $"BeaconBlock {beaconBlock!.ToString(Block.Format.FullHashAndNumber)} result {insertResult}");
                     }
 
                     headers = _chainLevelHelper!.GetNextHeaders(maxCount, maxHeaderNumber, 0);
@@ -635,6 +641,20 @@ public ScenarioBuilder AssertChainLevel(int startNumber, int finalNumber)
                 return this;
             }
 
+            public ScenarioBuilder AssertForceNewBeaconSync()
+            {
+                _beaconPivot.ShouldForceStartNewSync.Should().BeTrue();
+
+                return this;
+            }
+
+            public ScenarioBuilder AssertNotForceNewBeaconSync()
+            {
+                _beaconPivot.ShouldForceStartNewSync.Should().BeFalse();
+
+                return this;
+            }
+
             public ScenarioBuilder print()
             {
                 // Console.WriteLine("LowestInsertedBeaconHeader:"+_notSyncedTree!.LowestInsertedBeaconHeader.Number);

diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs b/src/Nethermind/Nethermind.Merge.Plugin/Handlers/V1/ForkchoiceUpdatedV1Handler.cs
@@ -126,6 +126,16 @@ public Task<ResultWrapper<ForkchoiceUpdatedV1Result>> Handle(ForkchoiceStateV1 f
                     return ForkchoiceUpdatedV1Result.Syncing;
                 }
 
+                if (_beaconPivot.ShouldForceStartNewSync)
+                {
+                    if (_logger.IsInfo)
+                        _logger.Info($"Force starting new sync.");
+
+                    StartNewBeaconHeaderSync(forkchoiceState, newHeadBlock!, requestStr);
+
+                    return ForkchoiceUpdatedV1Result.Syncing;
+                }
+
                 if (!blockInfo.IsBeaconMainChain && blockInfo.IsBeaconInfo)
                     ReorgBeaconChainDuringSync(newHeadBlock!, blockInfo);
 

diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/BeaconPivot.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/BeaconPivot.cs
@@ -73,6 +73,7 @@ public BeaconPivot(
         public Keccak PivotHash => CurrentBeaconPivot?.Hash ?? _syncConfig.PivotHashParsed;
 
         public BlockHeader? ProcessDestination { get; set; }
+        public bool ShouldForceStartNewSync { get; set; } = false;
 
         // We actually start beacon header sync from the pivot parent hash because hive test.... And because
         // we can I guess?
@@ -132,6 +133,7 @@ public void EnsurePivot(BlockHeader? blockHeader, bool updateOnlyIfNull = false)
                     BlockTreeInsertHeaderOptions.BeaconHeaderInsert | BlockTreeInsertHeaderOptions.TotalDifficultyNotNeeded);
                 CurrentBeaconPivot = blockHeader;
                 _blockTree.LowestInsertedBeaconHeader = blockHeader;
+                ShouldForceStartNewSync = false;
                 if (_logger.IsInfo) _logger.Info($"New beacon pivot: {blockHeader}");
             }
         }
@@ -174,5 +176,6 @@ public interface IBeaconPivot : IPivot
         // as MergeBlockDownloader process higher block, making it somewhat like a lowest processed beacon block.
         // TODO: Check if we can just re-use pivot and move pivot forward
         BlockHeader? ProcessDestination { get; set; }
+        bool ShouldForceStartNewSync { get; set; }
     }
 }
diff --git a/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/ChainLevelHelper.cs b/src/Nethermind/Nethermind.Merge.Plugin/Synchronization/ChainLevelHelper.cs
@@ -52,6 +52,17 @@ public ChainLevelHelper(
         _logger = logManager.GetClassLogger();
     }
 
+    private void OnMissingBeaconHeader(long blockNumber)
+    {
+        if (_beaconPivot.ProcessDestination?.Number > blockNumber)
+        {
+            // For some reason, this block number is missing when it should not.
+            // anyway, lets just restart the whole sync.
+            if (_logger.IsWarn) _logger.Warn($"Unable to find beacon header at height {blockNumber}. This is unexpected, forcing a new beacon sync.");
+            _beaconPivot.ShouldForceStartNewSync = true;
+        }
+    }
+
     public BlockHeader[]? GetNextHeaders(int maxCount, long maxHeaderNumber, int skipLastBlockCount = 0)
     {
         long? startingPoint = GetStartingPoint();
@@ -74,6 +85,7 @@ public ChainLevelHelper(
             BlockInfo? beaconMainChainBlock = level?.BeaconMainChainBlock;
             if (level == null || beaconMainChainBlock == null)
             {
+                OnMissingBeaconHeader(startingPoint.Value);
                 if (_logger.IsTrace)
                     _logger.Trace($"ChainLevelHelper.GetNextHeaders - level {startingPoint} not found");
                 break;
@@ -84,6 +96,7 @@ public ChainLevelHelper(
 
             if (newHeader == null)
             {
+                OnMissingBeaconHeader(startingPoint.Value);
                 if (_logger.IsTrace) _logger.Trace($"ChainLevelHelper - header {startingPoint} not found");
                 break;
             }
@@ -180,7 +193,11 @@ public bool TrySetNextBlocks(int maxCount, BlockDownloadContext context)
         if (_logger.IsTrace) _logger.Trace($"ChainLevelHelper. starting point's starting point is {startingPoint}. Best known number: {_blockTree.BestKnownNumber}, Process destination: {_beaconPivot.ProcessDestination?.Number}");
 
         BlockInfo? beaconMainChainBlock = GetBeaconMainChainBlockInfo(startingPoint);
-        if (beaconMainChainBlock == null) return null;
+        if (beaconMainChainBlock == null)
+        {
+            OnMissingBeaconHeader(startingPoint);
+            return null;
+        }
 
         if (!beaconMainChainBlock.IsBeaconInfo)
         {
@@ -201,6 +218,7 @@ public bool TrySetNextBlocks(int maxCount, BlockDownloadContext context)
             BlockInfo? parentBlockInfo = (_blockTree.GetInfo(header.Number - 1, header.ParentHash!)).Info;
             if (parentBlockInfo == null)
             {
+                OnMissingBeaconHeader(header.Number);
                 return null;
             }