Extend OperationTracer with new methods

CHANGELOG.md

@@ -1,29 +1,35 @@
+
 # Changelog
 
-## 23.4.4
+## 23.4.5
 
 ### Breaking Changes
 
+- Removed support for version 0 of the database as it is no longer used by any active node.
+
 ### Additions and Improvements
 - EvmTool now executes the `execution-spec-tests` via the `t8n` and `b11r`. See the [README](ethereum/evmtool/README.md) in EvmTool for more instructions.
 - Improve lifecycle management of the transaction pool [#5634](https://github.com/hyperledger/besu/pull/5634)
+- Add extension points in AbstractCreateOperation for EVM libraries to react to contract creations [#5656](https://github.com/hyperledger/besu/pull/5656)
+- Update to Tuweni 2.4.2. [#5684](https://github.com/hyperledger/besu/pull/5684)
 
 ### Bug Fixes
 - Use the node's configuration to determine if DNS enode URLs are allowed in calls to `admin_addPeer` and `admin_removePeer` [#5584](https://github.com/hyperledger/besu/pull/5584)
 - Align the implementation of Eth/68 `NewPooledTransactionHashes` to other clients, using unsigned int for encoding size. [#5640](https://github.com/hyperledger/besu/pull/5640)
 - Failure at startup when enabling layered txpool before initial sync done [#5636](https://github.com/hyperledger/besu/issues/5636)
+- Remove miner-related option warnings if the change isn't using Ethash consensus algorithm [#5669](https://github.com/hyperledger/besu/pull/5669)
 
 ### Download Links
 
 ---
 
-## 23.4.3
+## 23.4.4
 
 ### Breaking Changes
 - Move blockchain related variables in a dedicated storage, to pave the way to future optimizations [#5471](https://github.com/hyperledger/besu/pull/5471). The migration is performed automatically at startup,
 and in case a rollback is needed, before installing a previous version, the migration can be reverted, using the subcommand `storage revert-variables` with the same configuration use to run Besu.
 - Remove deprecated Rinkeby named network. [#5540](https://github.com/hyperledger/besu/pull/5540)
-
+- Use BlobDB for blockchain storage to reduce initial sync time and write amplification (PR #5475). This PR reduces sync time by 14 hours on m6a.xlarge VM (1 day 8 hours 27 minutes instead of 1 day 22 hours 4 minutes).
 ### Additions and Improvements
 - Allow Ethstats connection url to specify ws:// or wss:// scheme. [#5494](https://github.com/hyperledger/besu/issues/5494)
 - Add support for Shanghai changes to the GraphQL service [#5496](https://github.com/hyperledger/besu/pull/5496)
@@ -40,12 +46,16 @@ and in case a rollback is needed, before installing a previous version, the migr
 - Fix the unavailability of account code and storage on GraphQL/Bonsai [#5548](https://github.com/hyperledger/besu/pull/5548)
 
 ### Download Links
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.4.4/besu-23.4.4.tar.gz / sha256: bd476d235b6fe1f236a62bc709f41c87deb68b72c47bb5b58e56b9d9283af2c4
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/23.4.4/besu-23.4.4.zip / sha256: 4575000f4fd21d318e7b77340c9281d496bc800bee5b45a13684319e6f28bf27
 
-## 23.4.2
+## 23.4.3
 
 - Was not released (failed burn-in test)
 
----
+- ## 23.4.2
+
+- Was not released (failed burn-in test)
 
 ## 23.4.1
 

acceptance-tests/dsl/build.gradle

@@ -36,9 +36,9 @@ dependencies {
   implementation 'io.vertx:vertx-core'
   implementation 'junit:junit'
   implementation 'io.opentelemetry:opentelemetry-api'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-io'
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-io'
+  implementation 'io.tmio:tuweni-units'
   implementation 'org.assertj:assertj-core'
   implementation 'org.awaitility:awaitility'
   implementation 'org.java-websocket:Java-WebSocket'

acceptance-tests/tests/build.gradle

@@ -76,7 +76,7 @@ dependencies {
   testImplementation 'junit:junit'
   testImplementation 'org.apache.commons:commons-compress'
   testImplementation 'org.apache.logging.log4j:log4j-core'
-  testImplementation 'org.apache.tuweni:tuweni-crypto'
+  testImplementation 'io.tmio:tuweni-crypto'
   testImplementation 'org.assertj:assertj-core'
   testImplementation 'org.awaitility:awaitility'
   testImplementation 'org.junit.jupiter:junit-jupiter'

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/backup/BackupRoundTripAcceptanceTest.java

@@ -80,16 +80,6 @@ public BackupRoundTripAcceptanceTest(
   public static Object[][] getParameters() {
     return new Object[][] {
       // First 10 blocks of ropsten
-      new Object[] {
-        "Before versioning was enabled",
-        "version0",
-        0xA,
-        singletonList(
-            new AccountData(
-                "0xd1aeb42885a43b72b518182ef893125814811048",
-                BigInteger.valueOf(0xA),
-                Wei.fromHexString("0x2B5E3AF16B1880000"))),
-      },
       new Object[] {
         "After versioning was enabled and using multiple RocksDB columns",
         "version1",

acceptance-tests/tests/src/test/java/org/hyperledger/besu/tests/acceptance/database/DatabaseMigrationAcceptanceTest.java

@@ -56,16 +56,6 @@ public DatabaseMigrationAcceptanceTest(
   public static Object[][] getParameters() {
     return new Object[][] {
       // First 10 blocks of ropsten
-      new Object[] {
-        "Before versioning was enabled",
-        "version0",
-        0xA,
-        singletonList(
-            new AccountData(
-                "0xd1aeb42885a43b72b518182ef893125814811048",
-                BigInteger.valueOf(0xA),
-                Wei.fromHexString("0x2B5E3AF16B1880000"))),
-      },
       new Object[] {
         "After versioning was enabled and using multiple RocksDB columns",
         "version1",

besu/build.gradle

@@ -70,10 +70,10 @@ dependencies {
   implementation 'io.vertx:vertx-web'
   implementation 'org.apache.commons:commons-lang3'
   implementation 'org.apache.logging.log4j:log4j-core'
-  implementation 'org.apache.tuweni:tuweni-bytes'
-  implementation 'org.apache.tuweni:tuweni-config'
-  implementation 'org.apache.tuweni:tuweni-toml'
-  implementation 'org.apache.tuweni:tuweni-units'
+  implementation 'io.tmio:tuweni-bytes'
+  implementation 'io.tmio:tuweni-config'
+  implementation 'io.tmio:tuweni-toml'
+  implementation 'io.tmio:tuweni-units'
   implementation 'org.springframework.security:spring-security-crypto'
   implementation 'org.xerial.snappy:snappy-java'
   implementation 'tech.pegasys:jc-kzg-4844'
@@ -92,8 +92,8 @@ dependencies {
   testImplementation 'io.opentelemetry:opentelemetry-api'
   testImplementation 'junit:junit'
   testImplementation 'org.apache.commons:commons-text'
-  testImplementation 'org.apache.tuweni:tuweni-bytes'
-  testImplementation 'org.apache.tuweni:tuweni-units'
+  testImplementation 'io.tmio:tuweni-bytes'
+  testImplementation 'io.tmio:tuweni-units'
   testImplementation 'org.assertj:assertj-core'
   testImplementation 'org.awaitility:awaitility'
   testImplementation 'org.junit.jupiter:junit-jupiter'

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -2133,7 +2133,7 @@ private void issueOptionWarnings() {
             "--remote-connections-max-percentage"));
 
     // Check that block producer options work
-    if (!isMergeEnabled()) {
+    if (!isMergeEnabled() && getActualGenesisConfigOptions().isEthHash()) {
       CommandLineUtils.checkOptionDependencies(
           logger,
           commandLine,
@@ -2144,17 +2144,18 @@ private void issueOptionWarnings() {
               "--min-gas-price",
               "--min-block-occupancy-ratio",
               "--miner-extra-data"));
+
+      // Check that mining options are able to work
+      CommandLineUtils.checkOptionDependencies(
+          logger,
+          commandLine,
+          "--miner-enabled",
+          !minerOptionGroup.isMiningEnabled,
+          asList(
+              "--miner-stratum-enabled",
+              "--Xminer-remote-sealers-limit",
+              "--Xminer-remote-sealers-hashrate-ttl"));
     }
-    // Check that mining options are able to work
-    CommandLineUtils.checkOptionDependencies(
-        logger,
-        commandLine,
-        "--miner-enabled",
-        !minerOptionGroup.isMiningEnabled,
-        asList(
-            "--miner-stratum-enabled",
-            "--Xminer-remote-sealers-limit",
-            "--Xminer-remote-sealers-hashrate-ttl"));
 
     CommandLineUtils.failIfOptionDoesntMeetRequirement(
         commandLine,
@@ -3675,6 +3676,10 @@ private String generateConfigurationOverview() {
       builder.setHighSpecEnabled();
     }
 
+    if (buildTransactionPoolConfiguration().getLayeredTxPoolEnabled()) {
+      builder.setLayeredTxPoolEnabled();
+    }
+
     return builder.build();
   }
 }

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -47,6 +47,7 @@ public class ConfigurationOverviewBuilder {
   private Collection<String> engineApis;
   private String engineJwtFilePath;
   private boolean isHighSpec = false;
+  private boolean isLayeredTxPool = false;
   private Map<String, String> environment;
 
   /**
@@ -165,6 +166,16 @@ public ConfigurationOverviewBuilder setHighSpecEnabled() {
     return this;
   }
 
+  /**
+   * Sets experimental layered txpool enabled.
+   *
+   * @return the builder
+   */
+  public ConfigurationOverviewBuilder setLayeredTxPoolEnabled() {
+    isLayeredTxPool = true;
+    return this;
+  }
+
   /**
    * Sets the engine jwt file path.
    *
@@ -237,7 +248,11 @@ public String build() {
     }
 
     if (isHighSpec) {
-      lines.add("High spec configuration enabled");
+      lines.add("Experimental high spec configuration enabled");
+    }
+
+    if (isLayeredTxPool) {
+      lines.add("Experimental layered transaction pool configuration enabled");
     }
 
     lines.add("");

besu/src/main/java/org/hyperledger/besu/controller/CliqueBesuControllerBuilder.java

@@ -83,7 +83,7 @@ protected MiningCoordinator createMiningCoordinator(
         new CliqueMinerExecutor(
             protocolContext,
             protocolSchedule,
-            transactionPool.getPendingTransactions(),
+            transactionPool,
             nodeKey,
             miningParameters,
             new CliqueBlockScheduler(

besu/src/main/java/org/hyperledger/besu/controller/IbftBesuControllerBuilder.java

@@ -146,7 +146,7 @@ protected MiningCoordinator createMiningCoordinator(
     final BftProtocolSchedule bftProtocolSchedule = (BftProtocolSchedule) protocolSchedule;
     final BftBlockCreatorFactory<?> blockCreatorFactory =
         new BftBlockCreatorFactory<>(
-            transactionPool.getPendingTransactions(),
+            transactionPool,
             protocolContext,
             bftProtocolSchedule,
             forksSchedule,
@@ -310,7 +310,7 @@ private static MinedBlockObserver blockLogger(
                 block.getHeader().getCoinbase().equals(localAddress) ? "Produced" : "Imported",
                 block.getHeader().getNumber(),
                 block.getBody().getTransactions().size(),
-                transactionPool.getPendingTransactions().size(),
+                transactionPool.count(),
                 block.getHeader().getGasUsed(),
                 (block.getHeader().getGasUsed() * 100.0) / block.getHeader().getGasLimit(),
                 block.getHash().toHexString()));

besu/src/main/java/org/hyperledger/besu/controller/MainnetBesuControllerBuilder.java

@@ -49,7 +49,7 @@ protected MiningCoordinator createMiningCoordinator(
         new PoWMinerExecutor(
             protocolContext,
             protocolSchedule,
-            transactionPool.getPendingTransactions(),
+            transactionPool,
             miningParameters,
             new DefaultBlockScheduler(
                 MainnetBlockHeaderValidator.MINIMUM_SECONDS_SINCE_PARENT,

besu/src/main/java/org/hyperledger/besu/controller/MergeBesuControllerBuilder.java

@@ -174,7 +174,7 @@ protected MiningCoordinator createTransitionMiningCoordinator(
           LOG.debug("Block builder executor status {}", blockBuilderExecutor);
           return CompletableFuture.runAsync(task, blockBuilderExecutor);
         },
-        transactionPool.getPendingTransactions(),
+        transactionPool,
         miningParameters,
         backwardSyncContext,
         depositContractAddress);

besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java

@@ -184,7 +184,7 @@ protected MiningCoordinator createMiningCoordinator(
     final BftProtocolSchedule bftProtocolSchedule = (BftProtocolSchedule) protocolSchedule;
     final BftBlockCreatorFactory<?> blockCreatorFactory =
         new QbftBlockCreatorFactory(
-            transactionPool.getPendingTransactions(),
+            transactionPool,
             protocolContext,
             bftProtocolSchedule,
             qbftForksSchedule,
@@ -381,7 +381,7 @@ private static MinedBlockObserver blockLogger(
                 block.getHeader().getCoinbase().equals(localAddress) ? "Produced" : "Imported",
                 block.getHeader().getNumber(),
                 block.getBody().getTransactions().size(),
-                transactionPool.getPendingTransactions().size(),
+                transactionPool.count(),
                 block.getHeader().getGasUsed(),
                 (block.getHeader().getGasUsed() * 100.0) / block.getHeader().getGasLimit(),
                 block.getHash().toHexString()));

besu/src/main/java/org/hyperledger/besu/services/TraceServiceImpl.java

@@ -30,9 +30,9 @@
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSpec;
 import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
 import org.hyperledger.besu.ethereum.vm.CachingBlockHashLookup;
-import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.plugin.Unstable;
 import org.hyperledger.besu.plugin.services.TraceService;
+import org.hyperledger.besu.plugin.services.tracer.BlockAwareOperationTracer;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -68,7 +68,7 @@ public TraceServiceImpl(
    * @param tracer an instance of OperationTracer
    */
   @Override
-  public void traceBlock(final long blockNumber, final OperationTracer tracer) {
+  public void traceBlock(final long blockNumber, final BlockAwareOperationTracer tracer) {
     checkArgument(tracer != null);
     final Optional<Block> block = blockchainQueries.getBlockchain().getBlockByNumber(blockNumber);
     block.ifPresent(value -> trace(value, tracer));
@@ -81,13 +81,13 @@ public void traceBlock(final long blockNumber, final OperationTracer tracer) {
    * @param tracer an instance of OperationTracer
    */
   @Override
-  public void traceBlock(final Hash hash, final OperationTracer tracer) {
+  public void traceBlock(final Hash hash, final BlockAwareOperationTracer tracer) {
     checkArgument(tracer != null);
     final Optional<Block> block = blockchainQueries.getBlockchain().getBlockByHash(hash);
     block.ifPresent(value -> trace(value, tracer));
   }
 
-  private void trace(final Block block, final OperationTracer tracer) {
+  private void trace(final Block block, final BlockAwareOperationTracer tracer) {
     LOG.debug("Tracing block {}", block.toLogString());
     final List<TransactionProcessingResult> results = new ArrayList<>();
     Tracer.processTracing(
@@ -100,6 +100,9 @@ private void trace(final Block block, final OperationTracer tracer) {
           final MainnetTransactionProcessor transactionProcessor =
               protocolSpec.getTransactionProcessor();
           final BlockHeader header = block.getHeader();
+
+          tracer.traceStartBlock(block.getHeader(), block.getBody());
+
           block
               .getBody()
               .getTransactions()
@@ -114,6 +117,9 @@ private void trace(final Block block, final OperationTracer tracer) {
                                 maybeParentHeader
                                     .flatMap(BlockHeader::getExcessDataGas)
                                     .orElse(DataGas.ZERO));
+
+                    tracer.traceStartTransaction(transaction.getGasLimit());
+
                     final TransactionProcessingResult result =
                         transactionProcessor.processTransaction(
                             blockchain,
@@ -125,9 +131,15 @@ private void trace(final Block block, final OperationTracer tracer) {
                             new CachingBlockHashLookup(header, blockchain),
                             false,
                             dataGasPrice);
+
+                    long transactionGasUsed = transaction.getGasLimit() - result.getGasRemaining();
+                    tracer.traceEndTransaction(result.getOutput(), transactionGasUsed, 0);
+
                     results.add(result);
                   });
           return Optional.of(results);
         });
+
+    tracer.traceEndBlock(block.getHeader(), block.getBody());
   }
 }