Changes to allow evmtool t8n-server to work with execution-spec-tests

CHANGELOG.md

@@ -8,6 +8,7 @@
 - Removed support for version 0 of the database as it is no longer used by any active node.
 
 ### Additions and Improvements
+- `evmtool` launcher binaries now ship as part of the standard distribution. [#5701](https://github.com/hyperledger/besu/pull/5701) 
 - EvmTool now executes the `execution-spec-tests` via the `t8n` and `b11r`. See the [README](ethereum/evmtool/README.md) in EvmTool for more instructions.
 - Improve lifecycle management of the transaction pool [#5634](https://github.com/hyperledger/besu/pull/5634)
 - Add extension points in AbstractCreateOperation for EVM libraries to react to contract creations [#5656](https://github.com/hyperledger/besu/pull/5656)

build.gradle

@@ -559,7 +559,7 @@ task evmToolStartScripts(type: CreateStartScripts) {
   mainClass = 'org.hyperledger.besu.evmtool.EvmTool'
   classpath = startScripts.classpath
   outputDir = startScripts.outputDir
-  applicationName = 'evm'
+  applicationName = 'evmtool'
   defaultJvmOpts = [
     "-Dsecp256k1.randomize=false"
   ]
@@ -586,10 +586,10 @@ task autocomplete(type: JavaExec) {
   }
 }
 
-installDist { dependsOn checkLicenses }
+installDist { dependsOn checkLicenses, evmToolStartScripts }
 
 distTar {
-  dependsOn checkLicenses, autocomplete
+  dependsOn checkLicenses, autocomplete, evmToolStartScripts
   doFirst {
     delete fileTree(dir: 'build/distributions', include: '*.tar.gz')
   }
@@ -598,7 +598,7 @@ distTar {
 }
 
 distZip {
-  dependsOn checkLicenses, autocomplete
+  dependsOn checkLicenses, autocomplete, evmToolStartScripts
   doFirst {
     delete fileTree(dir: 'build/distributions', include: '*.zip')
   }
@@ -1021,6 +1021,7 @@ tasks.register("verifyDistributions") {
 
 dependencies {
   implementation project(':besu')
+  implementation project(':ethereum:evmtool')
   errorprone 'com.google.errorprone:error_prone_core'
 }
 

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/B11rSubCommand.java

@@ -23,6 +23,7 @@
 import org.hyperledger.besu.ethereum.mainnet.MainnetBlockHeaderFunctions;
 import org.hyperledger.besu.ethereum.referencetests.BlockchainReferenceTestCaseSpec.ReferenceTestBlockHeader;
 import org.hyperledger.besu.ethereum.rlp.BytesValueRLPOutput;
+import org.hyperledger.besu.util.LogConfigurator;
 
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
@@ -152,6 +153,7 @@ public B11rSubCommand(final EvmToolCommand parentCommand) {
 
   @Override
   public void run() {
+    LogConfigurator.setLevel("", "OFF");
     ObjectMapper objectMapper = JsonUtils.createObjectMapper();
     final ObjectReader b11rReader = objectMapper.reader();
 

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/EvmToolCommand.java

@@ -43,6 +43,7 @@
 import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.metrics.MetricsSystemModule;
+import org.hyperledger.besu.util.LogConfigurator;
 
 import java.io.BufferedWriter;
 import java.io.ByteArrayInputStream;
@@ -283,6 +284,7 @@ private static void addForkHelp(final CommandLine subCommandLine) {
 
   @Override
   public void run() {
+    LogConfigurator.setLevel("", "OFF");
     try {
       final EvmToolComponent component =
           DaggerEvmToolComponent.builder()

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/StateTestSubCommand.java

@@ -44,6 +44,7 @@
 import org.hyperledger.besu.evm.worldstate.WorldState;
 import org.hyperledger.besu.evm.worldstate.WorldUpdater;
 import org.hyperledger.besu.evmtool.exception.UnsupportedForkException;
+import org.hyperledger.besu.util.LogConfigurator;
 
 import java.io.BufferedReader;
 import java.io.File;
@@ -98,6 +99,7 @@ public StateTestSubCommand() {
 
   @Override
   public void run() {
+    LogConfigurator.setLevel("", "OFF");
     final ObjectMapper stateTestMapper = JsonUtils.createObjectMapper();
 
     final JavaType javaType =

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nExecutor.java

@@ -15,6 +15,9 @@
  */
 package org.hyperledger.besu.evmtool;
 
+import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_BASE;
+import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_PROTECTED_V_MIN;
+import static org.hyperledger.besu.ethereum.core.Transaction.REPLAY_UNPROTECTED_V_BASE;
 import static org.hyperledger.besu.ethereum.referencetests.ReferenceTestProtocolSchedules.shouldClearEmptyAccounts;
 
 import org.hyperledger.besu.config.StubGenesisConfigOptions;
@@ -104,6 +107,9 @@ protected static List<Transaction> extractTransactions(
           } else {
             Transaction.Builder builder = Transaction.builder();
             int type = Bytes.fromHexStringLenient(txNode.get("type").textValue()).toInt();
+            BigInteger chainId =
+                Bytes.fromHexStringLenient(txNode.get("chainId").textValue())
+                    .toUnsignedBigInteger();
             TransactionType transactionType = TransactionType.of(type == 0 ? 0xf8 : type);
             builder.type(transactionType);
             builder.nonce(Bytes.fromHexStringLenient(txNode.get("nonce").textValue()).toLong());
@@ -129,16 +135,11 @@ protected static List<Transaction> extractTransactions(
             if (txNode.has("to")) {
               builder.to(Address.fromHexString(txNode.get("to").textValue()));
             }
-
-            if (transactionType.requiresChainId()
-                || !txNode.has("protected")
-                || txNode.get("protected").booleanValue()) {
+            BigInteger v =
+                Bytes.fromHexStringLenient(txNode.get("v").textValue()).toUnsignedBigInteger();
+            if (transactionType.requiresChainId() || (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0)) {
               // chainid if protected
-              builder.chainId(
-                  new BigInteger(
-                      1,
-                      Bytes.fromHexStringLenient(txNode.get("chainId").textValue())
-                          .toArrayUnsafe()));
+              builder.chainId(chainId);
             }
 
             if (txNode.has("accessList")) {
@@ -190,12 +191,14 @@ protected static List<Transaction> extractTransactions(
 
               transactions.add(builder.signAndBuild(keys));
             } else {
-              BigInteger v =
-                  Bytes.fromHexStringLenient(txNode.get("v").textValue()).toUnsignedBigInteger();
-              if (v.compareTo(BigInteger.valueOf(35)) >= 0) {
-                v = v.subtract(BigInteger.valueOf(35)).mod(BigInteger.TWO);
-              } else if (v.compareTo(BigInteger.valueOf(27)) >= 0) {
-                v = v.subtract(BigInteger.valueOf(27)).mod(BigInteger.TWO);
+              if (transactionType == TransactionType.FRONTIER) {
+                if (v.compareTo(REPLAY_PROTECTED_V_MIN) > 0) {
+                  v =
+                      v.subtract(REPLAY_PROTECTED_V_BASE)
+                          .subtract(chainId.multiply(BigInteger.TWO));
+                } else {
+                  v = v.subtract(REPLAY_UNPROTECTED_V_BASE);
+                }
               }
               builder.signature(
                   SignatureAlgorithmFactory.getInstance()

ethereum/evmtool/src/main/java/org/hyperledger/besu/evmtool/T8nServerSubCommand.java

@@ -21,6 +21,7 @@
 import org.hyperledger.besu.ethereum.core.Transaction;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv;
 import org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState;
+import org.hyperledger.besu.evm.EvmSpecVersion;
 import org.hyperledger.besu.evm.tracing.OperationTracer;
 import org.hyperledger.besu.evmtool.T8nExecutor.RejectedTransaction;
 import org.hyperledger.besu.util.LogConfigurator;
@@ -33,11 +34,14 @@
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectReader;
+import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
 import io.vertx.core.Vertx;
-import io.vertx.core.json.JsonArray;
-import io.vertx.core.json.JsonObject;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpServerOptions;
+import io.vertx.core.http.HttpServerRequest;
 import picocli.CommandLine;
 
 @CommandLine.Command(
@@ -60,95 +64,132 @@ public class T8nServerSubCommand implements Runnable {
   public void run() {
     LogConfigurator.setLevel("", "OFF");
     Vertx.vertx()
-        .createHttpServer()
-        .requestHandler(
-            req ->
-                req.bodyHandler(
-                    body -> {
-                      ObjectMapper objectMapper = JsonUtils.createObjectMapper();
-                      JsonObject t8nRequest = body.toJsonObject();
-                      JsonObject state = t8nRequest.getJsonObject("state");
-                      String fork = state.getString("fork");
-                      Long chainId = Long.valueOf(state.getString("chainid"));
-                      String reward = state.getString("reward");
-
-                      JsonObject input = t8nRequest.getJsonObject("input");
-                      ReferenceTestEnv referenceTestEnv =
-                          input.getJsonObject("env").mapTo(ReferenceTestEnv.class);
-                      ReferenceTestWorldState initialWorldState =
-                          input.getJsonObject("alloc").mapTo(ReferenceTestWorldState.class);
-                      initialWorldState.persist(null);
-                      List<Transaction> transactions = new ArrayList<>();
-                      List<RejectedTransaction> rejections = new ArrayList<>();
-                      Object txs = input.getValue("txs");
-                      if (txs != null) {
-                        if (txs instanceof JsonArray txsArray) {
-                          extractTransactions(
-                              new PrintWriter(System.err, true, StandardCharsets.UTF_8),
-                              txsArray.stream().map(s -> (JsonNode) s).iterator(),
-                              transactions,
-                              rejections);
-                        } else if (txs instanceof String tx) {
-                          transactions =
-                              extractTransactions(
-                                  new PrintWriter(System.err, true, StandardCharsets.UTF_8),
-                                  List.<JsonNode>of(new TextNode(removeSurrounding("\"", tx)))
-                                      .iterator(),
-                                  transactions,
-                                  rejections);
-                        }
-                      }
-
-                      final T8nExecutor.T8nResult result =
-                          T8nExecutor.runTest(
-                              chainId,
-                              fork,
-                              reward,
-                              objectMapper,
-                              referenceTestEnv,
-                              initialWorldState,
-                              transactions,
-                              rejections,
-                              new T8nExecutor.TracerManager() {
-                                @Override
-                                public OperationTracer getManagedTracer(
-                                    final int txIndex, final Hash txHash) {
-                                  return OperationTracer.NO_TRACING;
-                                }
-
-                                @Override
-                                public void disposeTracer(final OperationTracer tracer) {
-                                  // No output streams to dispose of
-                                }
-                              });
-
-                      ObjectNode outputObject = objectMapper.createObjectNode();
-                      outputObject.set("alloc", result.allocObject());
-                      outputObject.set("body", result.bodyBytes());
-                      outputObject.set("result", result.resultObject());
-
-                      try {
-                        req.response()
-                            .putHeader("Content-Type", "application/json")
-                            .end(
-                                objectMapper
-                                    .writerWithDefaultPrettyPrinter()
-                                    .writeValueAsString(outputObject));
-                      } catch (JsonProcessingException e) {
-                        req.response().setStatusCode(500).end(e.getMessage());
-                      }
-                    }))
-        .listen(port, host)
+        .createHttpServer(
+            new HttpServerOptions()
+                .setHost(host)
+                .setPort(port)
+                .setHandle100ContinueAutomatically(true)
+                .setCompressionSupported(true))
+        .requestHandler(req -> req.bodyHandler(body -> handle(req, body)))
+        .listen()
         .onSuccess(
             server -> System.out.println("Transition server listening on " + server.actualPort()))
         .onFailure(
             err -> System.err.println("Failed to start transition server: " + err.getMessage()));
   }
 
-  private static String removeSurrounding(final String delimiter, final String message) {
-    if (message.startsWith(delimiter) && message.endsWith(delimiter)) {
-      return message.substring(delimiter.length(), message.length() - delimiter.length());
+  void handle(final HttpServerRequest req, final Buffer body) {
+    ObjectMapper objectMapper = JsonUtils.createObjectMapper();
+    final ObjectReader t8nReader = objectMapper.reader();
+    try {
+      var t8nRequest = t8nReader.readTree(body.toString());
+      JsonNode state = t8nRequest.get("state");
+      JsonNode input = t8nRequest.get("input");
+
+      if (state != null && input != null) {
+        handleT8nRequest(req, objectMapper, state, input);
+      } else {
+        sendHelp(req, objectMapper);
+      }
+      req.response().send();
+    } catch (JsonProcessingException e) {
+      req.response().setStatusCode(500).end(e.getMessage());
+    }
+  }
+
+  void handleT8nRequest(
+      final HttpServerRequest req,
+      final ObjectMapper objectMapper,
+      final JsonNode state,
+      final JsonNode input) {
+    try {
+      String fork = state.get("fork").asText();
+      Long chainId = Long.valueOf(state.get("chainid").asText());
+      String reward = state.get("reward").asText();
+
+      ReferenceTestEnv referenceTestEnv =
+          objectMapper.convertValue(input.get("env"), ReferenceTestEnv.class);
+      ReferenceTestWorldState initialWorldState =
+          objectMapper.convertValue(input.get("alloc"), ReferenceTestWorldState.class);
+      initialWorldState.persist(null);
+      List<Transaction> transactions = new ArrayList<>();
+      List<RejectedTransaction> rejections = new ArrayList<>();
+      JsonNode txs = input.get("txs");
+      if (txs != null) {
+        if (txs instanceof ArrayNode txsArray) {
+          extractTransactions(
+              new PrintWriter(System.err, true, StandardCharsets.UTF_8),
+              txsArray.elements(),
+              transactions,
+              rejections);
+        } else if (txs instanceof TextNode txt) {
+          transactions =
+              extractTransactions(
+                  new PrintWriter(System.err, true, StandardCharsets.UTF_8),
+                  List.<JsonNode>of(txt).iterator(),
+                  transactions,
+                  rejections);
+        }
+      }
+
+      final T8nExecutor.T8nResult result =
+          T8nExecutor.runTest(
+              chainId,
+              fork,
+              reward,
+              objectMapper,
+              referenceTestEnv,
+              initialWorldState,
+              transactions,
+              rejections,
+              new T8nExecutor.TracerManager() {
+                @Override
+                public OperationTracer getManagedTracer(final int txIndex, final Hash txHash) {
+                  return OperationTracer.NO_TRACING;
+                }
+
+                @Override
+                public void disposeTracer(final OperationTracer tracer) {
+                  // No output streams to dispose of
+                }
+              });
+
+      ObjectNode outputObject = objectMapper.createObjectNode();
+      outputObject.set("alloc", result.allocObject());
+      outputObject.set("body", result.bodyBytes());
+      outputObject.set("result", result.resultObject());
+
+      try {
+        String response =
+            objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(outputObject);
+        req.response().setChunked(true);
+        req.response().putHeader("Content-Type", "application/json").send(response);
+      } catch (JsonProcessingException e) {
+        req.response().setStatusCode(500).end(e.getMessage());
+      }
+    } catch (Throwable t) {
+      t.printStackTrace();
+      throw t;
+    }
+  }
+
+  private void sendHelp(final HttpServerRequest req, final ObjectMapper objectMapper) {
+    ObjectNode outputObject = objectMapper.createObjectNode();
+    outputObject.set("version", TextNode.valueOf(new VersionProvider().getVersion()[0]));
+    ArrayNode forks = objectMapper.createArrayNode();
+    outputObject.set("forks", forks);
+    for (var fork : EvmSpecVersion.values()) {
+      forks.add(TextNode.valueOf(fork.getName()));
+    }
+    outputObject.set("error", TextNode.valueOf("Both 'state' and 'input' fields must be set"));
+
+    try {
+      req.response()
+          .putHeader("Content-Type", "application/json")
+          .end(objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(outputObject));
+
+    } catch (JsonProcessingException e) {
+      req.response().setStatusCode(500).end(e.getMessage());
     }
-    return message;
   }
 }