[NC-2233] Randomly perform full validation when fast syncing blocks

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/DefaultSynchronizer.java

@@ -27,6 +27,8 @@
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
 import tech.pegasys.pantheon.ethereum.worldstate.WorldStateStorage;
 import tech.pegasys.pantheon.metrics.LabelledMetric;
+import tech.pegasys.pantheon.metrics.MetricCategory;
+import tech.pegasys.pantheon.metrics.MetricsSystem;
 import tech.pegasys.pantheon.metrics.OperationTimer;
 import tech.pegasys.pantheon.services.queue.InMemoryBigQueue;
 import tech.pegasys.pantheon.util.ExceptionUtils;
@@ -56,10 +58,14 @@ public DefaultSynchronizer(
       final WorldStateStorage worldStateStorage,
       final EthContext ethContext,
       final SyncState syncState,
-      final LabelledMetric<OperationTimer> ethTasksTimer) {
+      final MetricsSystem metricsSystem) {
     this.syncConfig = syncConfig;
     this.ethContext = ethContext;
     this.syncState = syncState;
+
+    final LabelledMetric<OperationTimer> ethTasksTimer =
+        metricsSystem.createLabelledTimer(
+            MetricCategory.SYNCHRONIZER, "task", "Internal processing tasks", "taskName");
     this.blockPropagationManager =
         new BlockPropagationManager<>(
             syncConfig,
@@ -94,7 +100,12 @@ public DefaultSynchronizer(
                       protocolContext,
                       ethContext,
                       syncState,
-                      ethTasksTimer),
+                      ethTasksTimer,
+                      metricsSystem.createLabelledCounter(
+                          MetricCategory.SYNCHRONIZER,
+                          "fast_sync_validation_mode",
+                          "Number of blocks validated using light vs full validation during fast sync",
+                          "validationMode")),
                   worldStateDownloader));
     } else {
       this.fastSyncDownloader = Optional.empty();

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/ValidationPolicy.java

@@ -0,0 +1,20 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.ethereum.eth.sync;
+
+import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode;
+
+public interface ValidationPolicy {
+
+  HeaderValidationMode getValidationModeForNextBlock();
+}

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncActions.java

@@ -23,6 +23,7 @@
 import tech.pegasys.pantheon.ethereum.eth.sync.state.SyncState;
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.WaitForPeersTask;
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
+import tech.pegasys.pantheon.metrics.Counter;
 import tech.pegasys.pantheon.metrics.LabelledMetric;
 import tech.pegasys.pantheon.metrics.OperationTimer;
 import tech.pegasys.pantheon.util.ExceptionUtils;
@@ -43,20 +44,23 @@ public class FastSyncActions<C> {
   private final EthContext ethContext;
   private final SyncState syncState;
   private final LabelledMetric<OperationTimer> ethTasksTimer;
+  private final LabelledMetric<Counter> fastSyncValidationCounter;
 
   public FastSyncActions(
       final SynchronizerConfiguration syncConfig,
       final ProtocolSchedule<C> protocolSchedule,
       final ProtocolContext<C> protocolContext,
       final EthContext ethContext,
       final SyncState syncState,
-      final LabelledMetric<OperationTimer> ethTasksTimer) {
+      final LabelledMetric<OperationTimer> ethTasksTimer,
+      final LabelledMetric<Counter> fastSyncValidationCounter) {
     this.syncConfig = syncConfig;
     this.protocolSchedule = protocolSchedule;
     this.protocolContext = protocolContext;
     this.ethContext = ethContext;
     this.syncState = syncState;
     this.ethTasksTimer = ethTasksTimer;
+    this.fastSyncValidationCounter = fastSyncValidationCounter;
   }
 
   public CompletableFuture<Void> waitForSuitablePeers() {
@@ -157,6 +161,7 @@ public CompletableFuture<Void> downloadChain(final FastSyncState currentState) {
             ethContext,
             syncState,
             ethTasksTimer,
+            fastSyncValidationCounter,
             currentState.getPivotBlockHeader().get());
     return downloader.start();
   }

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncBlockHandler.java

@@ -21,11 +21,11 @@
 import tech.pegasys.pantheon.ethereum.core.TransactionReceipt;
 import tech.pegasys.pantheon.ethereum.eth.manager.AbstractPeerTask.PeerTaskResult;
 import tech.pegasys.pantheon.ethereum.eth.manager.EthContext;
+import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy;
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.CompleteBlocksTask;
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.GetReceiptsFromPeerTask;
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.PipelinedImportChainSegmentTask.BlockHandler;
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.exceptions.InvalidBlockException;
-import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode;
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSpec;
 import tech.pegasys.pantheon.metrics.LabelledMetric;
@@ -46,16 +46,19 @@ public class FastSyncBlockHandler<C> implements BlockHandler<BlockWithReceipts>
   private final ProtocolContext<C> protocolContext;
   private final EthContext ethContext;
   private final LabelledMetric<OperationTimer> ethTasksTimer;
+  private final ValidationPolicy validationPolicy;
 
   public FastSyncBlockHandler(
       final ProtocolSchedule<C> protocolSchedule,
       final ProtocolContext<C> protocolContext,
       final EthContext ethContext,
-      final LabelledMetric<OperationTimer> ethTasksTimer) {
+      final LabelledMetric<OperationTimer> ethTasksTimer,
+      final ValidationPolicy validationPolicy) {
     this.protocolSchedule = protocolSchedule;
     this.protocolContext = protocolContext;
     this.ethContext = ethContext;
     this.ethTasksTimer = ethTasksTimer;
+    this.validationPolicy = validationPolicy;
   }
 
   @Override
@@ -105,7 +108,7 @@ public CompletableFuture<List<BlockWithReceipts>> validateAndImportBlocks(
           protocolContext,
           block,
           blockWithReceipt.getReceipts(),
-          HeaderValidationMode.LIGHT_SKIP_DETACHED)) {
+          validationPolicy.getValidationModeForNextBlock())) {
         return invalidBlockFailure(block);
       }
     }

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncChainDownloader.java

@@ -24,6 +24,7 @@
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.PipelinedImportChainSegmentTask;
 import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode;
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
+import tech.pegasys.pantheon.metrics.Counter;
 import tech.pegasys.pantheon.metrics.LabelledMetric;
 import tech.pegasys.pantheon.metrics.OperationTimer;
 
@@ -38,6 +39,7 @@ public class FastSyncChainDownloader<C> {
   private final ProtocolContext<C> protocolContext;
   private final EthContext ethContext;
   private final LabelledMetric<OperationTimer> ethTasksTimer;
+  private final LabelledMetric<Counter> fastSyncValidationCounter;
 
   FastSyncChainDownloader(
       final SynchronizerConfiguration config,
@@ -46,12 +48,14 @@ public class FastSyncChainDownloader<C> {
       final EthContext ethContext,
       final SyncState syncState,
       final LabelledMetric<OperationTimer> ethTasksTimer,
+      final LabelledMetric<Counter> fastSyncValidationCounter,
       final BlockHeader pivotBlockHeader) {
     this.config = config;
     this.protocolSchedule = protocolSchedule;
     this.protocolContext = protocolContext;
     this.ethContext = ethContext;
     this.ethTasksTimer = ethTasksTimer;
+    this.fastSyncValidationCounter = fastSyncValidationCounter;
     chainDownloader =
         new ChainDownloader<>(
             config,
@@ -86,6 +90,19 @@ private CompletableFuture<List<Block>> importBlocksForCheckpoints(
     if (checkpointHeaders.size() < 2) {
       return CompletableFuture.completedFuture(emptyList());
     }
+    final FastSyncValidationPolicy attachedValidationPolicy =
+        new FastSyncValidationPolicy(
+            config.fastSyncFullValidationRate(),
+            HeaderValidationMode.LIGHT_SKIP_DETACHED,
+            HeaderValidationMode.SKIP_DETACHED,
+            fastSyncValidationCounter);
+    final FastSyncValidationPolicy detatchedValidationPolicy =
+        new FastSyncValidationPolicy(
+            config.fastSyncFullValidationRate(),
+            HeaderValidationMode.LIGHT_DETACHED_ONLY,
+            HeaderValidationMode.DETACHED_ONLY,
+            fastSyncValidationCounter);
+
     final PipelinedImportChainSegmentTask<C, BlockWithReceipts> importTask =
         PipelinedImportChainSegmentTask.forCheckpoints(
             protocolSchedule,
@@ -94,8 +111,12 @@ private CompletableFuture<List<Block>> importBlocksForCheckpoints(
             config.downloaderParallelism(),
             ethTasksTimer,
             new FastSyncBlockHandler<>(
-                protocolSchedule, protocolContext, ethContext, ethTasksTimer),
-            HeaderValidationMode.LIGHT_DETACHED_ONLY,
+                protocolSchedule,
+                protocolContext,
+                ethContext,
+                ethTasksTimer,
+                attachedValidationPolicy),
+            detatchedValidationPolicy,
             checkpointHeaders);
     return importTask
         .run()

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncValidationPolicy.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.ethereum.eth.sync.fastsync;
+
+import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy;
+import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode;
+import tech.pegasys.pantheon.metrics.Counter;
+import tech.pegasys.pantheon.metrics.LabelledMetric;
+
+public class FastSyncValidationPolicy implements ValidationPolicy {
+  private final float targetFullValidationRate;
+  private final HeaderValidationMode lightValidationMode;
+  private final HeaderValidationMode fullValidationMode;
+  private final LabelledMetric<Counter> fastSyncValidationCounter;
+
+  public FastSyncValidationPolicy(
+      final float targetFullValidationRate,
+      final HeaderValidationMode lightValidationMode,
+      final HeaderValidationMode fullValidationMode,
+      final LabelledMetric<Counter> fastSyncValidationCounter) {
+    this.targetFullValidationRate = targetFullValidationRate;
+    this.lightValidationMode = lightValidationMode;
+    this.fullValidationMode = fullValidationMode;
+    this.fastSyncValidationCounter = fastSyncValidationCounter;
+  }
+
+  @Override
+  public HeaderValidationMode getValidationModeForNextBlock() {
+    final HeaderValidationMode mode =
+        Math.random() < targetFullValidationRate ? fullValidationMode : lightValidationMode;
+    fastSyncValidationCounter.labels(mode.name()).inc();
+    return mode;
+  }
+}

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/fullsync/FullSyncDownloader.java

@@ -102,7 +102,7 @@ private CompletableFuture<List<Block>> importBlocksForCheckpoints(
               ethTasksTimer,
               new FullSyncBlockHandler<>(
                   protocolSchedule, protocolContext, ethContext, ethTasksTimer),
-              HeaderValidationMode.DETACHED_ONLY,
+              () -> HeaderValidationMode.DETACHED_ONLY,
               checkpointHeaders);
       importedBlocks = importTask.run();
     }

ethereum/eth/src/main/java/tech/pegasys/pantheon/ethereum/eth/sync/tasks/PipelinedImportChainSegmentTask.java

@@ -16,9 +16,9 @@
 import tech.pegasys.pantheon.ethereum.core.BlockHeader;
 import tech.pegasys.pantheon.ethereum.eth.manager.AbstractEthTask;
 import tech.pegasys.pantheon.ethereum.eth.manager.EthContext;
+import tech.pegasys.pantheon.ethereum.eth.sync.ValidationPolicy;
 import tech.pegasys.pantheon.ethereum.eth.sync.tasks.exceptions.InvalidBlockException;
 import tech.pegasys.pantheon.ethereum.mainnet.BlockHeaderValidator;
-import tech.pegasys.pantheon.ethereum.mainnet.HeaderValidationMode;
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSchedule;
 import tech.pegasys.pantheon.ethereum.mainnet.ProtocolSpec;
 import tech.pegasys.pantheon.metrics.LabelledMetric;
@@ -51,7 +51,7 @@ public class PipelinedImportChainSegmentTask<C, B> extends AbstractEthTask<List<
   private final List<BlockHeader> checkpointHeaders;
   private final int chunksInTotal;
   private final BlockHandler<B> blockHandler;
-  private final HeaderValidationMode headerValidationMode;
+  private final ValidationPolicy validationPolicy;
   private int chunksIssued;
   private int chunksCompleted;
   private final int maxActiveChunks;
@@ -71,7 +71,7 @@ protected PipelinedImportChainSegmentTask(
       final List<BlockHeader> checkpointHeaders,
       final LabelledMetric<OperationTimer> ethTasksTimer,
       final BlockHandler<B> blockHandler,
-      final HeaderValidationMode headerValidationMode) {
+      final ValidationPolicy validationPolicy) {
     super(ethTasksTimer);
     this.protocolSchedule = protocolSchedule;
     this.protocolContext = protocolContext;
@@ -80,10 +80,10 @@ protected PipelinedImportChainSegmentTask(
     this.checkpointHeaders = checkpointHeaders;
     this.chunksInTotal = checkpointHeaders.size() - 1;
     this.blockHandler = blockHandler;
+    this.validationPolicy = validationPolicy;
     this.chunksIssued = 0;
     this.chunksCompleted = 0;
     this.maxActiveChunks = maxActiveChunks;
-    this.headerValidationMode = headerValidationMode;
   }
 
   public static <C, B> PipelinedImportChainSegmentTask<C, B> forCheckpoints(
@@ -93,7 +93,7 @@ public static <C, B> PipelinedImportChainSegmentTask<C, B> forCheckpoints(
       final int maxActiveChunks,
       final LabelledMetric<OperationTimer> ethTasksTimer,
       final BlockHandler<B> blockHandler,
-      final HeaderValidationMode headerValidationMode,
+      final ValidationPolicy validationPolicy,
       final BlockHeader... checkpointHeaders) {
     return forCheckpoints(
         protocolSchedule,
@@ -102,7 +102,7 @@ public static <C, B> PipelinedImportChainSegmentTask<C, B> forCheckpoints(
         maxActiveChunks,
         ethTasksTimer,
         blockHandler,
-        headerValidationMode,
+        validationPolicy,
         Arrays.asList(checkpointHeaders));
   }
 
@@ -113,7 +113,7 @@ public static <C, B> PipelinedImportChainSegmentTask<C, B> forCheckpoints(
       final int maxActiveChunks,
       final LabelledMetric<OperationTimer> ethTasksTimer,
       final BlockHandler<B> blockHandler,
-      final HeaderValidationMode headerValidationMode,
+      final ValidationPolicy validationPolicy,
       final List<BlockHeader> checkpointHeaders) {
     return new PipelinedImportChainSegmentTask<>(
         protocolSchedule,
@@ -123,7 +123,7 @@ public static <C, B> PipelinedImportChainSegmentTask<C, B> forCheckpoints(
         checkpointHeaders,
         ethTasksTimer,
         blockHandler,
-        headerValidationMode);
+        validationPolicy);
   }
 
   @Override
@@ -241,7 +241,10 @@ private CompletableFuture<List<BlockHeader>> validateHeaders(final List<BlockHea
           final BlockHeaderValidator<C> blockHeaderValidator =
               protocolSpec.getBlockHeaderValidator();
           if (blockHeaderValidator.validateHeader(
-              childHeader, parentHeader, protocolContext, headerValidationMode)) {
+              childHeader,
+              parentHeader,
+              protocolContext,
+              validationPolicy.getValidationModeForNextBlock())) {
             // The first header will be imported by the previous request range.
             result.complete(headers.subList(1, headers.size()));
           } else {

ethereum/eth/src/test/java/tech/pegasys/pantheon/ethereum/eth/sync/fastsync/FastSyncActionsTest.java

@@ -16,6 +16,7 @@
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 import static tech.pegasys.pantheon.ethereum.eth.sync.fastsync.FastSyncError.CHAIN_TOO_SHORT;
 import static tech.pegasys.pantheon.ethereum.eth.sync.fastsync.FastSyncError.NO_PEERS_AVAILABLE;
+import static tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem.NO_OP_LABELLED_COUNTER;
 import static tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem.NO_OP_LABELLED_TIMER;
 
 import tech.pegasys.pantheon.ethereum.ProtocolContext;
@@ -78,7 +79,8 @@ public void setUp() {
             protocolContext,
             ethContext,
             new SyncState(blockchain, ethContext.getEthPeers()),
-            ethTasksTimer);
+            ethTasksTimer,
+            NO_OP_LABELLED_COUNTER);
   }
 
   @Test