Add support for mining ommers, up to 8 blocks behind the head job (hy…

…perledger#2576)

* Add support for mining ommers, up to 8 blocks behind the head job

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

* Add a CLI option to set PoW jobs TTL

Signed-off-by: Antoine Toulme <antoine@lunar-ocean.com>

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1722,7 +1722,8 @@ public BesuControllerBuilder getControllerBuilder() {
                 Optional.empty(),
                 minBlockOccupancyRatio,
                 unstableMiningOptions.getRemoteSealersLimit(),
-                unstableMiningOptions.getRemoteSealersTimeToLive()))
+                unstableMiningOptions.getRemoteSealersTimeToLive(),
+                unstableMiningOptions.getPowJobTimeToLive()))
         .transactionPoolConfiguration(buildTransactionPoolConfiguration())
         .nodeKey(buildNodeKey())
         .metricsSystem(metricsSystem.get())

besu/src/main/java/org/hyperledger/besu/cli/options/unstable/MiningOptions.java

@@ -14,6 +14,7 @@
  */
 package org.hyperledger.besu.cli.options.unstable;
 
+import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_POW_JOB_TTL;
 import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_REMOTE_SEALERS_LIMIT;
 import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_REMOTE_SEALERS_TTL;
 
@@ -35,6 +36,13 @@ public class MiningOptions {
           "Specifies the lifetime of each entry in the cache. An entry will be automatically deleted if no update has been received before the deadline (default: ${DEFAULT-VALUE} minutes)")
   private final Long remoteSealersTimeToLive = DEFAULT_REMOTE_SEALERS_TTL;
 
+  @CommandLine.Option(
+      hidden = true,
+      names = {"--Xminer-pow-job-ttl"},
+      description =
+          "Specifies the time PoW jobs are kept in cache and will accept a solution from miners (default: ${DEFAULT-VALUE} milliseconds)")
+  private final Long powJobTimeToLive = DEFAULT_POW_JOB_TTL;
+
   @SuppressWarnings({"FieldCanBeFinal", "FieldMayBeFinal"}) // PicoCLI requires non-final Strings.
   @CommandLine.Option(
       hidden = true,
@@ -57,4 +65,8 @@ public Long getRemoteSealersTimeToLive() {
   public String getStratumExtranonce() {
     return stratumExtranonce;
   }
+
+  public Long getPowJobTimeToLive() {
+    return powJobTimeToLive;
+  }
 }

besu/src/main/java/org/hyperledger/besu/cli/subcommands/blocks/BlocksSubCommand.java

@@ -16,6 +16,7 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.hyperledger.besu.cli.subcommands.blocks.BlocksSubCommand.COMMAND_NAME;
+import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_POW_JOB_TTL;
 import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_REMOTE_SEALERS_LIMIT;
 import static org.hyperledger.besu.ethereum.core.MiningParameters.DEFAULT_REMOTE_SEALERS_TTL;
 
@@ -269,7 +270,8 @@ private MiningParameters getMiningParameters() {
           Optional.of(new IncrementingNonceGenerator(0)),
           0.0,
           DEFAULT_REMOTE_SEALERS_LIMIT,
-          DEFAULT_REMOTE_SEALERS_TTL);
+          DEFAULT_REMOTE_SEALERS_TTL,
+          DEFAULT_POW_JOB_TTL);
     }
 
     private void importJsonBlocks(final BesuController controller, final Path path)

besu/src/main/java/org/hyperledger/besu/controller/MainnetBesuControllerBuilder.java

@@ -54,7 +54,8 @@ protected MiningCoordinator createMiningCoordinator(
                 MainnetBlockHeaderValidator.TIMESTAMP_TOLERANCE_S,
                 clock),
             gasLimitCalculator,
-            epochCalculator);
+            epochCalculator,
+            miningParameters.getPowJobTimeToLive());
 
     final PoWMiningCoordinator miningCoordinator =
         new PoWMiningCoordinator(

ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/PoWMinerExecutor.java

@@ -35,6 +35,7 @@ public class PoWMinerExecutor extends AbstractMinerExecutor<PoWBlockMiner> {
   protected boolean stratumMiningEnabled;
   protected final Iterable<Long> nonceGenerator;
   protected final EpochCalculator epochCalculator;
+  protected final long powJobTimeToLive;
 
   public PoWMinerExecutor(
       final ProtocolContext protocolContext,
@@ -43,7 +44,8 @@ public PoWMinerExecutor(
       final MiningParameters miningParams,
       final AbstractBlockScheduler blockScheduler,
       final GasLimitCalculator gasLimitCalculator,
-      final EpochCalculator epochCalculator) {
+      final EpochCalculator epochCalculator,
+      final long powJobTimeToLive) {
     super(
         protocolContext,
         protocolSchedule,
@@ -54,6 +56,7 @@ public PoWMinerExecutor(
     this.coinbase = miningParams.getCoinbase();
     this.nonceGenerator = miningParams.getNonceGenerator().orElse(new RandomNonceGenerator());
     this.epochCalculator = epochCalculator;
+    this.powJobTimeToLive = powJobTimeToLive;
   }
 
   @Override
@@ -78,7 +81,8 @@ public PoWBlockMiner createMiner(
             protocolSchedule.getByBlockNumber(parentHeader.getNumber() + 1).getPoWHasher().get(),
             stratumMiningEnabled,
             ethHashObservers,
-            epochCalculator);
+            epochCalculator,
+            powJobTimeToLive);
     final Function<BlockHeader, PoWBlockCreator> blockCreator =
         (header) ->
             new PoWBlockCreator(

ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/PoWBlockCreatorTest.java

@@ -85,7 +85,8 @@ public void createMainnetBlock1() throws IOException {
             PoWHasher.ETHASH_LIGHT,
             false,
             Subscribers.none(),
-            new EpochCalculator.DefaultEpochCalculator());
+            new EpochCalculator.DefaultEpochCalculator(),
+            1000);
 
     final PendingTransactions pendingTransactions =
         new PendingTransactions(
@@ -144,7 +145,8 @@ public void createMainnetBlock1_fixedDifficulty1() {
             PoWHasher.ETHASH_LIGHT,
             false,
             Subscribers.none(),
-            new EpochCalculator.DefaultEpochCalculator());
+            new EpochCalculator.DefaultEpochCalculator(),
+            1000);
 
     final PendingTransactions pendingTransactions =
         new PendingTransactions(
@@ -198,7 +200,8 @@ public void rewardBeneficiary_zeroReward_skipZeroRewardsFalse() {
             PoWHasher.ETHASH_LIGHT,
             false,
             Subscribers.none(),
-            new EpochCalculator.DefaultEpochCalculator());
+            new EpochCalculator.DefaultEpochCalculator(),
+            1000);
 
     final PendingTransactions pendingTransactions =
         new PendingTransactions(
@@ -268,7 +271,8 @@ public void rewardBeneficiary_zeroReward_skipZeroRewardsTrue() {
             PoWHasher.ETHASH_LIGHT,
             false,
             Subscribers.none(),
-            new EpochCalculator.DefaultEpochCalculator());
+            new EpochCalculator.DefaultEpochCalculator(),
+            1000);
 
     final PendingTransactions pendingTransactions =
         new PendingTransactions(

ethereum/blockcreation/src/test/java/org/hyperledger/besu/ethereum/blockcreation/PoWMinerExecutorTest.java

@@ -59,7 +59,8 @@ public void startingMiningWithoutCoinbaseThrowsException() {
             miningParameters,
             new DefaultBlockScheduler(1, 10, TestClock.fixed()),
             GasLimitCalculator.constant(),
-            new EpochCalculator.DefaultEpochCalculator());
+            new EpochCalculator.DefaultEpochCalculator(),
+            1000);
 
     assertThatExceptionOfType(CoinbaseNotSetException.class)
         .isThrownBy(() -> executor.startAsyncMining(Subscribers.create(), Subscribers.none(), null))
@@ -88,7 +89,8 @@ public void settingCoinbaseToNullThrowsException() {
             miningParameters,
             new DefaultBlockScheduler(1, 10, TestClock.fixed()),
             GasLimitCalculator.constant(),
-            new EpochCalculator.DefaultEpochCalculator());
+            new EpochCalculator.DefaultEpochCalculator(),
+            1000);
 
     assertThatExceptionOfType(IllegalArgumentException.class)
         .isThrownBy(() -> executor.setCoinbase(null))

ethereum/core/build.gradle

@@ -46,6 +46,7 @@ dependencies {
   implementation 'net.java.dev.jna:jna'
   implementation 'org.apache.logging.log4j:log4j-api'
   implementation 'org.apache.tuweni:tuweni-bytes'
+  implementation 'org.apache.tuweni:tuweni-concurrent'
   implementation 'org.apache.tuweni:tuweni-units'
   implementation 'org.hyperledger.besu:bls12-381'
   implementation 'org.immutables:value-annotations'

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/MiningParameters.java

@@ -26,6 +26,8 @@ public class MiningParameters {
 
   public static final long DEFAULT_REMOTE_SEALERS_TTL = Duration.ofMinutes(10).toMinutes();
 
+  public static final long DEFAULT_POW_JOB_TTL = Duration.ofMinutes(5).toMillis();
+
   private final Optional<Address> coinbase;
   private final Wei minTransactionGasPrice;
   private final Bytes extraData;
@@ -38,6 +40,7 @@ public class MiningParameters {
   private final Double minBlockOccupancyRatio;
   private final int remoteSealersLimit;
   private final long remoteSealersTimeToLive;
+  private final long powJobTimeToLive;
 
   public MiningParameters(
       final Address coinbase,
@@ -56,7 +59,8 @@ public MiningParameters(
         Optional.empty(),
         0.8,
         DEFAULT_REMOTE_SEALERS_LIMIT,
-        DEFAULT_REMOTE_SEALERS_TTL);
+        DEFAULT_REMOTE_SEALERS_TTL,
+        DEFAULT_POW_JOB_TTL);
   }
 
   public MiningParameters(
@@ -71,7 +75,8 @@ public MiningParameters(
       final Optional<Iterable<Long>> maybeNonceGenerator,
       final Double minBlockOccupancyRatio,
       final int remoteSealersLimit,
-      final long remoteSealersTimeToLive) {
+      final long remoteSealersTimeToLive,
+      final long powJobTimeToLive) {
     this.coinbase = Optional.ofNullable(coinbase);
     this.minTransactionGasPrice = minTransactionGasPrice;
     this.extraData = extraData;
@@ -84,6 +89,7 @@ public MiningParameters(
     this.minBlockOccupancyRatio = minBlockOccupancyRatio;
     this.remoteSealersLimit = remoteSealersLimit;
     this.remoteSealersTimeToLive = remoteSealersTimeToLive;
+    this.powJobTimeToLive = powJobTimeToLive;
   }
 
   public Optional<Address> getCoinbase() {
@@ -134,6 +140,10 @@ public long getRemoteSealersTimeToLive() {
     return remoteSealersTimeToLive;
   }
 
+  public long getPowJobTimeToLive() {
+    return powJobTimeToLive;
+  }
+
   @Override
   public boolean equals(final Object o) {
     if (this == o) return true;
@@ -149,7 +159,8 @@ public boolean equals(final Object o) {
         && Objects.equals(stratumExtranonce, that.stratumExtranonce)
         && Objects.equals(minBlockOccupancyRatio, that.minBlockOccupancyRatio)
         && Objects.equals(remoteSealersTimeToLive, that.remoteSealersTimeToLive)
-        && Objects.equals(remoteSealersLimit, that.remoteSealersLimit);
+        && Objects.equals(remoteSealersLimit, that.remoteSealersLimit)
+        && Objects.equals(powJobTimeToLive, that.powJobTimeToLive);
   }
 
   @Override
@@ -165,7 +176,8 @@ public int hashCode() {
         stratumExtranonce,
         minBlockOccupancyRatio,
         remoteSealersLimit,
-        remoteSealersTimeToLive);
+        remoteSealersTimeToLive,
+        powJobTimeToLive);
   }
 
   @Override
@@ -197,6 +209,8 @@ public String toString() {
         + remoteSealersLimit
         + ", remoteSealersTimeToLive="
         + remoteSealersTimeToLive
+        + ", powJobTimeToLive="
+        + powJobTimeToLive
         + '}';
   }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/PoWSolver.java

@@ -26,11 +26,15 @@
 
 import com.google.common.base.Stopwatch;
 import org.apache.logging.log4j.Logger;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.concurrent.ExpiringMap;
 import org.apache.tuweni.units.bigints.UInt256;
 
 public class PoWSolver {
 
+  private static final int MAX_OMMER_DEPTH = 8;
   private static final Logger LOG = getLogger();
+  private final long powJobTimeToLive;
 
   public static class PoWSolverJob {
 
@@ -80,30 +84,35 @@ PoWSolution getSolution() throws InterruptedException, ExecutionException {
   private final Subscribers<PoWObserver> ethHashObservers;
   private final EpochCalculator epochCalculator;
   private volatile Optional<PoWSolverJob> currentJob = Optional.empty();
+  private final ExpiringMap<Bytes, PoWSolverJob> currentJobs = new ExpiringMap<>();
 
   public PoWSolver(
       final Iterable<Long> nonceGenerator,
       final PoWHasher poWHasher,
       final Boolean stratumMiningEnabled,
       final Subscribers<PoWObserver> ethHashObservers,
-      final EpochCalculator epochCalculator) {
+      final EpochCalculator epochCalculator,
+      final long powJobTimeToLive) {
     this.nonceGenerator = nonceGenerator;
     this.poWHasher = poWHasher;
     this.stratumMiningEnabled = stratumMiningEnabled;
     this.ethHashObservers = ethHashObservers;
     ethHashObservers.forEach(observer -> observer.setSubmitWorkCallback(this::submitSolution));
     this.epochCalculator = epochCalculator;
+    this.powJobTimeToLive = powJobTimeToLive;
   }
 
   public PoWSolution solveFor(final PoWSolverJob job)
       throws InterruptedException, ExecutionException {
     currentJob = Optional.of(job);
+    currentJobs.put(
+        job.getInputs().getPrePowHash(), job, System.currentTimeMillis() + powJobTimeToLive);
     if (stratumMiningEnabled) {
       ethHashObservers.forEach(observer -> observer.newJob(job.inputs));
     } else {
       findValidNonce();
     }
-    return currentJob.get().getSolution();
+    return job.getSolution();
   }
 
   private void findValidNonce() {
@@ -149,22 +158,50 @@ public Optional<Long> hashesPerSecond() {
 
   public boolean submitSolution(final PoWSolution solution) {
     final Optional<PoWSolverJob> jobSnapshot = currentJob;
+    PoWSolverJob jobToTestWith = null;
     if (jobSnapshot.isEmpty()) {
       LOG.debug("No current job, rejecting miner work");
       return false;
     }
 
-    final PoWSolverJob job = jobSnapshot.get();
-    final PoWSolverInputs inputs = job.getInputs();
-    if (!inputs.getPrePowHash().equals(solution.getPowHash())) {
-      LOG.debug("Miner's solution does not match current job");
+    PoWSolverJob headJob = jobSnapshot.get();
+    if (headJob.getInputs().getPrePowHash().equals(solution.getPowHash())) {
+      LOG.debug("Head job matches the solution pow hash {}", solution.getPowHash());
+      jobToTestWith = headJob;
+    }
+    if (jobToTestWith == null) {
+      PoWSolverJob ommerCandidate = currentJobs.get(solution.getPowHash());
+      if (ommerCandidate != null) {
+        long distanceToHead =
+            headJob.getInputs().getBlockNumber() - ommerCandidate.getInputs().getBlockNumber();
+        LOG.debug(
+            "Found ommer candidate {} with block number {}, distance to head {}",
+            solution.getPowHash(),
+            ommerCandidate.getInputs().getBlockNumber(),
+            distanceToHead);
+        if (distanceToHead <= MAX_OMMER_DEPTH) {
+          jobToTestWith = ommerCandidate;
+        } else {
+          LOG.debug("Discarded ommer solution as too far from head {}", distanceToHead);
+        }
+      }
+    }
+    if (jobToTestWith == null) {
+      LOG.debug("No matching job found for hash {}, rejecting solution", solution.getPowHash());
       return false;
     }
+    if (jobToTestWith.isDone()) {
+      LOG.debug("Matching job found for hash {}, but already solved", solution.getPowHash());
+      return false;
+    }
+    final PoWSolverInputs inputs = jobToTestWith.getInputs();
+
     final Optional<PoWSolution> calculatedSolution = testNonce(inputs, solution.getNonce());
 
     if (calculatedSolution.isPresent()) {
       LOG.debug("Accepting a solution from a miner");
-      currentJob.get().solvedWith(calculatedSolution.get());
+      currentJobs.remove(solution.getPowHash());
+      jobToTestWith.solvedWith(calculatedSolution.get());
       return true;
     }
     LOG.debug("Rejecting a solution from a miner");