Implement trace replay block transactions trace option (#1886)

* Factorise metrics code for KeyValueStorage database

- introduce `MonitorableKeyValueStorage`
- factorise code
- remove metrics instanciation in `RocksDbKeyValueStorage` and `ColumnarRocksDbKeyValueStorage`

* Rename class

* [PAN-2995] Make account.getAddress() return an Optional<Address>

* fix Jenkins failure

* fix failing test

* fix failing test

* Revert "fix failing test"

This reverts commit e2bb261.

* Revert "fix failing test"

This reverts commit e5dc3fa.

* Revert "fix Jenkins failure"

This reverts commit 15f90ea.

* Revert "[PAN-2995] Make account.getAddress() return an Optional<Address>"

This reverts commit 08cc7ca.

* init trace_replayBlockTransactions

* Revert "init trace_replayBlockTransactions"

This reverts commit 5454265.

* Revert "Revert "init trace_replayBlockTransactions""

This reverts commit 756f52b.

* Update TraceReplayBlockTransactions.java

* Check request parameters

Method returns an error if any option other than “trace” is supplied.

* Update TraceReplayBlockTransactions.java

* implement trace_replayBlockTransactions

- start business logic implementation
- iterate through `TraceFrame` and transform to parity style
- detect `CALL` opcode to retrieve contract address required in parity format

* remove comments

* add final modifier for method parameters

* fix unit tests

* remove useless log

* transform address from stack (Bytes32) to an Address

* trace_replayBlockTransactions new features

- compute gas used for each sub trace
- detect `RETURN` opcode
    - to get `ouput` value from `memory`
    - detect end of subtrace

* fix unit tests

* Update FlatTrace.java

add final modifier

* pretty format JSON file

* handle smart contract deployments

* handle smart contract deployment

- set `code` field when contract creation transaction
- set `address` field when contract creation transaction

* introduce FlatTraceGenerator

* refactoring

* spotless apply

* update tests to compare Json node tree independently of the order

- added generated test cases
- use jackson `ObjectMapper` to compare the actual result to the expected one
- `assertThat(mapper.readTree(actualResult)).isEqualTo(mapper.readTree(expectedResult));`

* fix unit tests

- return empty result if block is empty
- handle case where there is no `RETURN`, `STOP` is then considered as a return to close the trace and perform gas computation.
- increment gas remaining by gas used

* spotless apply

* introduce SELFDESTRUCT

* set result to null when suicide type

* remove result field, use builder only

* re-enable tracing tests

* implement gas used computation

- implement a strategy with 2 options
  - if first frame gas remaining is higher than gas remaining after transaction was processed then the gas used will be the net difference
  - otherwise a frame by frame cumulative gas cost will be used

* get self destruct refund balance from message frame

* give access to `ProtocolSchedule` to get `GasCalculator`

* fix gas used computation for subtrace (1 level deep)

* fix input, use all memory and not only first element

* fix trace addresses

* debug log

* remove some tests files

* Revert "remove some tests files"

This reverts commit 88ee6aa.

* debug jenkins

* out instead of err

* spotless apply

* add debug info

* remove static modifier for blockchainSetupUtil field

* remove unused field

* add javadoc for gasCalculator parameter

* fix PR review comments

* add javadoc for Trace interface

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/core/Gas.java

@@ -149,4 +149,8 @@ public boolean equals(final Object obj) {
   public String toString() {
     return Long.toString(value);
   }
+
+  public String toHexString() {
+    return String.format("0x%s", Long.toHexString(value));
+  }
 }

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/debug/TraceFrame.java

@@ -12,7 +12,9 @@
  */
 package tech.pegasys.pantheon.ethereum.debug;
 
+import tech.pegasys.pantheon.ethereum.core.Address;
 import tech.pegasys.pantheon.ethereum.core.Gas;
+import tech.pegasys.pantheon.ethereum.core.Wei;
 import tech.pegasys.pantheon.ethereum.vm.ExceptionalHaltReason;
 import tech.pegasys.pantheon.util.bytes.Bytes32;
 import tech.pegasys.pantheon.util.bytes.BytesValue;
@@ -36,6 +38,7 @@ public class TraceFrame {
   private final Optional<Bytes32[]> memory;
   private final Optional<Map<UInt256, UInt256>> storage;
   private final Optional<BytesValue> revertReason;
+  private final Optional<Map<Address, Wei>> maybeRefunds;
 
   public TraceFrame(
       final int pc,
@@ -47,7 +50,8 @@ public TraceFrame(
       final Optional<Bytes32[]> stack,
       final Optional<Bytes32[]> memory,
       final Optional<Map<UInt256, UInt256>> storage,
-      final Optional<BytesValue> revertReason) {
+      final Optional<BytesValue> revertReason,
+      final Optional<Map<Address, Wei>> maybeRefunds) {
     this.pc = pc;
     this.opcode = opcode;
     this.gasRemaining = gasRemaining;
@@ -58,6 +62,32 @@ public TraceFrame(
     this.memory = memory;
     this.storage = storage;
     this.revertReason = revertReason;
+    this.maybeRefunds = maybeRefunds;
+  }
+
+  public TraceFrame(
+      final int pc,
+      final String opcode,
+      final Gas gasRemaining,
+      final Optional<Gas> gasCost,
+      final int depth,
+      final EnumSet<ExceptionalHaltReason> exceptionalHaltReasons,
+      final Optional<Bytes32[]> stack,
+      final Optional<Bytes32[]> memory,
+      final Optional<Map<UInt256, UInt256>> storage,
+      final Optional<BytesValue> revertReason) {
+    this(
+        pc,
+        opcode,
+        gasRemaining,
+        gasCost,
+        depth,
+        exceptionalHaltReasons,
+        stack,
+        memory,
+        storage,
+        revertReason,
+        Optional.empty());
   }
 
   public TraceFrame(
@@ -123,6 +153,10 @@ public Optional<BytesValue> getRevertReason() {
     return revertReason;
   }
 
+  public Optional<Map<Address, Wei>> getMaybeRefunds() {
+    return maybeRefunds;
+  }
+
   @Override
   public String toString() {
     return MoreObjects.toStringHelper(this)

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/ProtocolSpec.java

@@ -19,13 +19,16 @@
 import tech.pegasys.pantheon.ethereum.core.Wei;
 import tech.pegasys.pantheon.ethereum.mainnet.MainnetBlockProcessor.TransactionReceiptFactory;
 import tech.pegasys.pantheon.ethereum.vm.EVM;
+import tech.pegasys.pantheon.ethereum.vm.GasCalculator;
 
 /** A protocol specification. */
 public class ProtocolSpec<C> {
 
   private final String name;
   private final EVM evm;
 
+  private final GasCalculator gasCalculator;
+
   private final TransactionValidator transactionValidator;
 
   private final TransactionProcessor transactionProcessor;
@@ -76,6 +79,7 @@ public class ProtocolSpec<C> {
    * @param miningBeneficiaryCalculator determines to whom mining proceeds are paid
    * @param precompileContractRegistry all the pre-compiled contracts added
    * @param skipZeroBlockRewards should rewards be skipped if it is zero
+   * @param gasCalculator the gas calculator to use.
    */
   public ProtocolSpec(
       final String name,
@@ -94,7 +98,8 @@ public ProtocolSpec(
       final Wei blockReward,
       final MiningBeneficiaryCalculator miningBeneficiaryCalculator,
       final PrecompileContractRegistry precompileContractRegistry,
-      final boolean skipZeroBlockRewards) {
+      final boolean skipZeroBlockRewards,
+      final GasCalculator gasCalculator) {
     this.name = name;
     this.evm = evm;
     this.transactionValidator = transactionValidator;
@@ -112,6 +117,7 @@ public ProtocolSpec(
     this.miningBeneficiaryCalculator = miningBeneficiaryCalculator;
     this.precompileContractRegistry = precompileContractRegistry;
     this.skipZeroBlockRewards = skipZeroBlockRewards;
+    this.gasCalculator = gasCalculator;
   }
 
   /**
@@ -259,6 +265,15 @@ public PrecompileContractRegistry getPrecompileContractRegistry() {
     return precompileContractRegistry;
   }
 
+  /**
+   * Returns the gasCalculator used in this specification.
+   *
+   * @return the gas calculator
+   */
+  public GasCalculator getGasCalculator() {
+    return gasCalculator;
+  }
+
   public void setTransactionFilter(final TransactionFilter transactionFilter) {
     transactionValidator.setTransactionFilter(transactionFilter);
   }

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/mainnet/ProtocolSpecBuilder.java

@@ -331,7 +331,8 @@ public ProtocolSpec<T> build(final ProtocolSchedule<T> protocolSchedule) {
         blockReward,
         miningBeneficiaryCalculator,
         precompileContractRegistry,
-        skipZeroBlockRewards);
+        skipZeroBlockRewards,
+        gasCalculator);
   }
 
   public interface TransactionProcessorBuilder {

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/vm/DebugOperationTracer.java

@@ -14,7 +14,9 @@
 
 import static tech.pegasys.pantheon.util.uint.UInt256.U_32;
 
+import tech.pegasys.pantheon.ethereum.core.Address;
 import tech.pegasys.pantheon.ethereum.core.Gas;
+import tech.pegasys.pantheon.ethereum.core.Wei;
 import tech.pegasys.pantheon.ethereum.debug.TraceFrame;
 import tech.pegasys.pantheon.ethereum.debug.TraceOptions;
 import tech.pegasys.pantheon.ethereum.vm.ehalt.ExceptionalHaltException;
@@ -56,7 +58,8 @@ public void traceExecution(
       executeOperation.execute();
     } finally {
       final Optional<Map<UInt256, UInt256>> storage = captureStorage(frame);
-
+      final Optional<Map<Address, Wei>> maybeRefunds =
+          frame.getRefunds().isEmpty() ? Optional.empty() : Optional.of(frame.getRefunds());
       traceFrames.add(
           new TraceFrame(
               pc,
@@ -68,7 +71,8 @@ public void traceExecution(
               stack,
               memory,
               storage,
-              frame.getRevertReason()));
+              frame.getRevertReason(),
+              maybeRefunds));
     }
   }
 

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/vm/MessageFrame.java

@@ -32,7 +32,9 @@
 
 import java.util.Deque;
 import java.util.EnumSet;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Consumer;
@@ -202,6 +204,7 @@ public enum Type {
   private final LogSeries logs;
   private Gas gasRefund;
   private final Set<Address> selfDestructs;
+  private final Map<Address, Wei> refunds;
 
   // Execution Environment fields.
   private final Address recipient;
@@ -271,6 +274,7 @@ private MessageFrame(
     this.logs = LogSeries.empty();
     this.gasRefund = Gas.ZERO;
     this.selfDestructs = new HashSet<>();
+    this.refunds = new HashMap<>();
     this.recipient = recipient;
     this.originator = originator;
     this.contract = contract;
@@ -654,6 +658,25 @@ public Set<Address> getSelfDestructs() {
     return selfDestructs;
   }
 
+  /**
+   * Add refund to the refunds map if not already present.
+   *
+   * @param beneficiary the beneficiary of the refund.
+   * @param amount the amount of the refund.
+   */
+  public void addRefund(final Address beneficiary, final Wei amount) {
+    refunds.put(beneficiary, amount);
+  }
+
+  /**
+   * Returns the refunds map.
+   *
+   * @return the refunds map
+   */
+  public Map<Address, Wei> getRefunds() {
+    return refunds;
+  }
+
   /**
    * Returns the current blockchain.
    *

ethereum/core/src/main/java/tech/pegasys/pantheon/ethereum/vm/operations/SelfDestructOperation.java

@@ -54,6 +54,10 @@ public void execute(final MessageFrame frame) {
         frame.getWorldState().getOrCreate(Words.toAddress(frame.popStackItem()));
 
     recipient.incrementBalance(account.getBalance());
+
+    // add refund in message frame
+    frame.addRefund(recipient.getAddress(), account.getBalance());
+
     account.setBalance(Wei.ZERO);
 
     frame.setState(MessageFrame.State.CODE_SUCCESS);

ethereum/jsonrpc/src/main/java/tech/pegasys/pantheon/ethereum/jsonrpc/JsonRpcMethodsFactory.java

@@ -79,6 +79,7 @@
 import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.NetServices;
 import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.NetVersion;
 import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.RpcModules;
+import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.TraceReplayBlockTransactions;
 import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.TxPoolPantheonStatistics;
 import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.TxPoolPantheonTransactions;
 import tech.pegasys.pantheon.ethereum.jsonrpc.internal.methods.Web3ClientVersion;
@@ -334,6 +335,20 @@ blockchainQueries, new TransactionTracer(blockReplay), parameter),
           new AdminChangeLogLevel(parameter));
     }
 
+    if (rpcApis.contains(RpcApis.TRACE)) {
+      addMethods(
+          enabledMethods,
+          new TraceReplayBlockTransactions(
+              parameter,
+              new BlockTracer(
+                  new BlockReplay(
+                      protocolSchedule,
+                      blockchainQueries.getBlockchain(),
+                      blockchainQueries.getWorldStateArchive())),
+              blockchainQueries,
+              protocolSchedule));
+    }
+
     final boolean eea = rpcApis.contains(RpcApis.EEA);
     final boolean priv = rpcApis.contains(RpcApis.PRIV);
     if (eea || priv) {

ethereum/jsonrpc/src/main/java/tech/pegasys/pantheon/ethereum/jsonrpc/RpcApis.java

@@ -28,6 +28,7 @@ public class RpcApis {
   public static final RpcApi EEA = new RpcApi("EEA");
   public static final RpcApi PRIV = new RpcApi("PRIV");
   public static final RpcApi TX_POOL = new RpcApi("TXPOOL");
+  public static final RpcApi TRACE = new RpcApi("TRACE");
 
   public static final List<RpcApi> DEFAULT_JSON_RPC_APIS = Arrays.asList(ETH, NET, WEB3);
 
@@ -52,6 +53,8 @@ public static Optional<RpcApi> valueOf(final String name) {
       return Optional.of(PRIV);
     } else if (name.equals(TX_POOL.getCliValue())) {
       return Optional.of(TX_POOL);
+    } else if (name.equals(TRACE.getCliValue())) {
+      return Optional.of(TRACE);
     } else {
       return Optional.empty();
     }