Update evmtool graalvm build (hyperledger#5639)

* add new class to reflection config
* move CLI output to be system-err oriented
* make logger in evmtool nop
* exclude some problematic jars from compilation
  - mixed versions of bouncy castle
  - netty initializations of log4j
* Standardize JSON ObjectMapper Construction
* Add some manual mappings
* Directly instantiate crypto algorithms where appropriate

Signed-off-by: Danno Ferrin <danno.ferrin@swirldslabs.com>

CHANGELOG.md

@@ -5,6 +5,7 @@
 ### Breaking Changes
 
 ### Additions and Improvements
+- EvmTool now executes the `execution-spec-tests` via the `t8n` and `b11r`. See the [README](ethereum/evmtool/README.md) in EvmTool for more instructions. 
 
 ### Bug Fixes
 - Use the node's configuration to determine if DNS enode URLs are allowed in calls to `admin_addPeer` and `admin_removePeer` [#5584](https://github.com/hyperledger/besu/pull/5584)

build.gradle

@@ -1021,7 +1021,6 @@ tasks.register("verifyDistributions") {
 
 dependencies {
   implementation project(':besu')
-  implementation project(':ethereum:evmtool')
   errorprone 'com.google.errorprone:error_prone_core'
 }
 

crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/Hash.java

@@ -14,6 +14,11 @@
  */
 package org.hyperledger.besu.crypto;
 
+import static org.hyperledger.besu.crypto.MessageDigestFactory.BLAKE2BF_ALG;
+import static org.hyperledger.besu.crypto.MessageDigestFactory.KECCAK256_ALG;
+import static org.hyperledger.besu.crypto.MessageDigestFactory.RIPEMD160_ALG;
+import static org.hyperledger.besu.crypto.MessageDigestFactory.SHA256_ALG;
+
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.function.Supplier;
@@ -26,13 +31,6 @@
 public abstract class Hash {
   private Hash() {}
 
-  /** The constant KECCAK256_ALG. */
-  public static final String KECCAK256_ALG = "KECCAK-256";
-
-  private static final String SHA256_ALG = "SHA-256";
-  private static final String RIPEMD160_ALG = "RIPEMD160";
-  private static final String BLAKE2BF_ALG = "BLAKE2BF";
-
   private static final Supplier<MessageDigest> KECCAK256_SUPPLIER =
       Suppliers.memoize(() -> messageDigest(KECCAK256_ALG));
   private static final Supplier<MessageDigest> SHA256_SUPPLIER =

crypto/algorithms/src/main/java/org/hyperledger/besu/crypto/MessageDigestFactory.java

@@ -18,11 +18,25 @@
 import java.security.NoSuchAlgorithmException;
 import java.security.Security;
 
+import org.bouncycastle.jcajce.provider.digest.Keccak;
+import org.bouncycastle.jcajce.provider.digest.RIPEMD160;
+import org.bouncycastle.jcajce.provider.digest.SHA256;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 
 /** The Message digest factory. */
 public class MessageDigestFactory {
 
+  private MessageDigestFactory() {}
+
+  /** Keccak-256 */
+  public static final String KECCAK256_ALG = "KECCAK-256";
+  /** SHA-256 */
+  public static final String SHA256_ALG = "SHA-256";
+  /** RipeMD-160 */
+  public static final String RIPEMD160_ALG = "RIPEMD160";
+  /** Blake2b F Function */
+  public static final String BLAKE2BF_ALG = "BLAKE2BF";
+
   static {
     Security.addProvider(new BesuProvider());
     Security.addProvider(new BouncyCastleProvider());
@@ -37,6 +51,12 @@ public class MessageDigestFactory {
    */
   @SuppressWarnings("DoNotInvokeMessageDigestDirectly")
   public static MessageDigest create(final String algorithm) throws NoSuchAlgorithmException {
-    return MessageDigest.getInstance(algorithm);
+    return switch (algorithm) {
+      case KECCAK256_ALG -> new Keccak.Digest256();
+      case SHA256_ALG -> new SHA256.Digest();
+      case RIPEMD160_ALG -> new RIPEMD160.Digest();
+      case BLAKE2BF_ALG -> new Blake2bfMessageDigest();
+      default -> MessageDigest.getInstance(algorithm);
+    };
   }
 }

enclave/src/integration-test/java/org/hyperledger/besu/enclave/TlsHelpers.java

@@ -39,6 +39,8 @@
 
 public class TlsHelpers {
 
+  private TlsHelpers() {}
+
   private static KeyStore loadP12KeyStore(final File pkcsFile, final String password)
       throws KeyStoreException, NoSuchAlgorithmException, CertificateException {
     final KeyStore store = KeyStore.getInstance("pkcs12");
@@ -82,7 +84,7 @@ public static List<X509Certificate> getCertsFromPkcs12(final TlsCertificateDefin
 
   private static String generateFingerprint(final X509Certificate cert)
       throws NoSuchAlgorithmException, CertificateEncodingException {
-    final MessageDigest md = MessageDigestFactory.create("SHA-256");
+    final MessageDigest md = MessageDigestFactory.create(MessageDigestFactory.SHA256_ALG);
     md.update(cert.getEncoded());
     final byte[] digest = md.digest();
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/mainnet/DirectAcyclicGraphSeed.java

@@ -14,9 +14,9 @@
  */
 package org.hyperledger.besu.ethereum.mainnet;
 
+import static org.hyperledger.besu.crypto.MessageDigestFactory.KECCAK256_ALG;
 import static org.hyperledger.besu.ethereum.mainnet.EthHash.EPOCH_LENGTH;
 
-import org.hyperledger.besu.crypto.Hash;
 import org.hyperledger.besu.crypto.MessageDigestFactory;
 
 import java.security.DigestException;
@@ -25,11 +25,13 @@
 
 public class DirectAcyclicGraphSeed {
 
+  private DirectAcyclicGraphSeed() {}
+
   public static final ThreadLocal<MessageDigest> KECCAK_256 =
       ThreadLocal.withInitial(
           () -> {
             try {
-              return MessageDigestFactory.create(Hash.KECCAK256_ALG);
+              return MessageDigestFactory.create(KECCAK256_ALG);
             } catch (final NoSuchAlgorithmException ex) {
               throw new IllegalStateException(ex);
             }

ethereum/evmtool/README.md

@@ -0,0 +1,127 @@
+EVM Tool
+========
+
+EVM Tool is a stand alone EVM executor and test execution tool. The
+principal purpose of the tool is for testing and validation of the EVM
+and enclosing data structures.
+
+Using EVM Tool in execution-specification-tests
+-----------------------------------------------
+
+To use EVM Tool in Execution Spec tests it is recommended that you use
+the GraalVM build, as the framework incurs significant startup penalties
+for each invocation when run via the Java runtime.
+
+### Building Execution Tests on macOS
+
+Current as of 24 Jun 2023.
+
+MacOS users will typically encounter two problems,one relating to the
+version of Python used and one relating to zsh.
+
+Homebrew will only install the most recent version of Python
+as `python3`, and that is 3.11. The execution tests require 3.10. The
+solution is to use a 3.10 version of python to set up the virtual
+environment.
+
+```zsh
+python3.10 -m venv ./venv/
+```
+
+Zsh requires braces to be escaped in the command line, so the step to
+install python packages needs to escape the brackets
+
+```zsh
+pip install -e .\[docs,lint,test\]
+```
+
+An all-in-one script, including homebrew, would look like
+
+```zsh
+brew install ethereum solidity
+git clone https://github.com/ethereum/execution-spec-tests
+cd execution-spec-tests
+python3 -m venv ./venv/
+source ./venv/bin/activate
+pip install -e .[docs,lint,test]
+```
+
+### Building EvmTool with GraalVM on macOS
+
+First you need a GraalVM JDK installed, if not already installed.
+
+It is recommended you install [SDKMAN](https://sdkman.io/install) to
+manage the graalVM install, homebrew has issues with native attributes
+and code signing.
+
+```zsh
+sdk install java 22.3.r17-grl 
+sdk use java 22.3.r17-grl
+```
+
+You can also manually install GraalVM from
+the [GraalVM website](https://www.graalvm.org/downloads)..
+
+Once GraalVM is installed you use the `nativeCompile` target.
+
+```zsh
+./gradlew nativeCompile 
+```
+
+The resulting binary
+is `./ethereum/evmtool/build/native/nativeCompile/evmtool`
+
+If the testing repository and besu are installed in the same parent
+directory, the command to run the execution tests is
+
+```zsh
+fill -v tests --evm-bin ../besu/ethereum/evmtool/build/install/evmtool/bin/evm 
+```
+
+Assuming homebrew and SDKMan are both installed, the complete script is
+
+```zsh
+sdk install java 22.3.r17-grl 
+sdk use java 22.3.r17-grl
+git clone https://github.com/hyperledger/besu
+cd besu
+./gradlew nativeCompile
+cd ..
+
+brew install ethereum solidity
+git clone https://github.com/ethereum/execution-spec-tests
+cd execution-spec-tests
+python3 -m venv ./venv/
+source ./venv/bin/activate
+pip install -e .[docs,lint,test]
+
+fill -v tests --evm-bin ../besu/ethereum/evmtool/build/install/evmtool/bin/evm
+```
+
+If you don't want to use the GraalVM tool the binary that is compatible
+is generated by the `ethereum:evmtool:installdist` target and is located
+at `./ethereum/evmtool/build/install/evmtool/bin/evm`
+
+Why not GraalVM for everything?
+-------------------------------
+
+Using GraalVM in execution-spec-tests results in over 20x performance
+increase in execution. It will be faster to build GraalVM from scratch
+and run the execution-spec-tests than to run just the Java version.
+
+It is demonstrably faster to run the Java version for a node.
+All the test execution gains are the result of reduced startup
+penalties. Larger benchmarks will show that performance intensive EVM
+code will be slower in GraalVM than the Java version due to the adaptive
+compiler.
+
+For contracts that execute 30 million gas in small operations it is
+often faster to run the Java EVMTool than the GraalVM EVMTool, including
+startup penalty. The execution tests focus on smaller VM tests that
+demonstrate specification conformance.
+
+We would also need to reconsider some library choices. GraalVM does not
+work with Log4J, and we would have to ban that library across Besu and
+all dependents. Libraries such as Netty also have some problematic entry
+points that interact poorly with how SLF4J needs to initialize.
+

ethereum/evmtool/build.gradle

@@ -56,6 +56,8 @@ dependencies {
   implementation 'info.picocli:picocli'
   implementation 'io.vertx:vertx-core'
 
+  runtimeOnly 'org.slf4j:slf4j-nop'
+
   annotationProcessor 'com.google.dagger:dagger-compiler'
   annotationProcessor 'info.picocli:picocli-codegen'
 
@@ -65,9 +67,6 @@ dependencies {
   testImplementation 'org.mockito:mockito-core'
 
   testRuntimeOnly 'org.junit.vintage:junit-vintage-engine'
-
-  // No logging in grallvm EvmTool
-  nativeImageClasspath 'org.slf4j:slf4j-nop'
 }
 
 mainClassName = 'org.hyperledger.besu.evmtool.EvmTool'
@@ -160,7 +159,7 @@ graalvmNative {
 
 
       // Netty drags in older versions of bouncy castle, exclude it so there are no conflicts
-      excludeConfig.put("io.netty:netty-buffer:4.1.74.Final", [".*"])
+      excludeConfig.put("io.netty:netty-buffer", [".*"])
     }
   }
 }
@@ -169,10 +168,12 @@ graalvmNative {
 configurations.nativeImageClasspath {
   // netty statically allocates some problematic classes
   exclude group: 'io.netty', module: 'netty-buffer'
+  exclude group: 'io.netty', module: 'netty-common'
   exclude group: 'io.netty', module: 'netty-transport'
 
   // keep log4j from sneaking in.  GraalVM has an aleric reaction if it sees even one class
   exclude group: 'org.slf4j', module: 'log4j-over-slf4j:1.7.36'
   exclude group: "log4j", module: "log4j"
   exclude group: "org.apache.logging.log4j"
+  exclude group: 'org.bouncycastle', module: 'bcprov-jdk18on'
 }

ethereum/evmtool/src/main/graal/reflection-config.json

@@ -64,41 +64,57 @@
   },
   {
     "name": "org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv",
-    "queryAllPublicConstructors" : true,
-    "queryAllPublicMethods" : true,
+    "queryAllPublicConstructors": true,
+    "queryAllPublicMethods": true,
     "allDeclaredConstructors": true,
     "allPublicMethods": true
   },
   {
     "name": "org.hyperledger.besu.ethereum.referencetests.ReferenceTestEnv$EnvWithdrawal",
-    "queryAllPublicConstructors" : true,
-    "queryAllPublicMethods" : true,
+    "queryAllPublicConstructors": true,
+    "queryAllPublicMethods": true,
     "allDeclaredConstructors": true,
     "allPublicMethods": true
   },
   {
     "name": "org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState",
-    "queryAllPublicConstructors" : true,
-    "queryAllPublicMethods" : true,
+    "queryAllPublicConstructors": true,
+    "queryAllPublicMethods": true,
     "allDeclaredConstructors": true,
     "allPublicMethods": true
   },
   {
     "name": "org.hyperledger.besu.ethereum.referencetests.ReferenceTestWorldState$AccountMock",
-    "queryAllPublicConstructors" : true,
-    "queryAllPublicMethods" : true,
+    "queryAllPublicConstructors": true,
+    "queryAllPublicMethods": true,
     "allDeclaredConstructors": true,
     "allPublicMethods": true
   },
   {
     "name": "org.hyperledger.besu.ethereum.referencetests.BlockchainReferenceTestCaseSpec$ReferenceTestBlockHeader",
-    "queryAllPublicConstructors" : true,
-    "queryAllPublicMethods" : true,
+    "queryAllPublicConstructors": true,
+    "queryAllPublicMethods": true,
     "allDeclaredConstructors": true,
     "allPublicMethods": true
   },
   {
     "name": "org.hyperledger.besu.evm.internal.ReturnStack$ReturnStackItem[]",
     "unsafeAllocated": true
+  },
+  {
+    "name": "org.hyperledger.besu.evmtool.T8nSubCommand$RejectedTransaction",
+    "queryAllPublicConstructors": true,
+    "queryAllPublicMethods": true,
+    "allDeclaredConstructors": true,
+    "allPublicMethods": true
+  },
+  {
+    "name": "org.hyperledger.besu.evm.log.Log",
+    "allDeclaredConstructors": true,
+    "allPublicConstructors": true,
+    "allDeclaredMethods": true,
+    "allPublicMethods": true,
+    "allDeclaredFields": true,
+    "allPublicFields": true
   }
 ]