Use Java 17 to build and run Besu (hyperledger#4977)

Update following sections:
Gradle build, CircleCI, GitHub Actions, Docker

The devp2p TLS mutual authentication unit tests dealing with PKCS11+nss and QBFT+PKCS11+nss ATs were failing on Java 17 when dealing with RSA keys. The unit tests and AT are updated to use EC keys instead. The keystores, truststores, CRL etc. involved in those ATs are regenerated and detailed instructions are provided on how they are created.

Signed-off-by: Usman Saleem <usman@usmans.info>

.circleci/config.yml

@@ -1,29 +1,29 @@
 ---
 version: 2.1
 orbs:
-  win: circleci/windows@2.2.0
+  win: circleci/windows@5.0
 
 executors:
   besu_executor_small:
     docker:
-      - image: cimg/openjdk:11.0
+      - image: cimg/openjdk:17.0
     resource_class: small
     working_directory: ~/project
     environment:
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2
 
   besu_executor_med: # 2cpu, 4G ram
     docker:
-      - image: cimg/openjdk:11.0
+      - image: cimg/openjdk:17.0
     resource_class: medium
     working_directory: ~/project
     environment:
       architecture: "amd64"
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=2
 
   besu_arm64_executor_med: # 2cpu, 8G ram
-    machine:
-      image: ubuntu-2004:202101-01
+    machine: #https://circleci.com/developer/machine/image/ubuntu-2204
+      image: ubuntu-2204:2022.10.2
     resource_class: arm.medium
     working_directory: ~/project
     environment:
@@ -32,15 +32,15 @@ executors:
 
   besu_executor_xl: # 8cpu, 16G ram
     docker:
-      - image: cimg/openjdk:11.0
+      - image: cimg/openjdk:17.0
     resource_class: xlarge
     working_directory: ~/project
     environment:
       GRADLE_OPTS: -Dorg.gradle.daemon=false -Dorg.gradle.parallel=true -Dorg.gradle.workers.max=4
 
   xl_machine_executor:
-    machine:
-      image: ubuntu-2004:202201-02 #Ubuntu 20.04, Docker v20.10.12, Docker Compose v1.29.2, Google Cloud SDK updates
+    machine: #https://circleci.com/developer/machine/image/ubuntu-2204
+      image: ubuntu-2204:2022.10.2
     resource_class: xlarge
 
   trivy_executor:
@@ -60,6 +60,7 @@ commands:
             sudo apt-get update
             sudo apt-get install -y libsodium23 libsodium-dev libjemalloc-dev apt-transport-https haveged libnss3-tools
             sudo service haveged restart
+            java --version
       - restore_gradle_cache
   restore_gradle_cache:
     description: "Restore Gradle cache"
@@ -269,13 +270,13 @@ jobs:
           at: ~/project
       - setup_remote_docker
       - run:
-          name: hadoLint_openjdk_11
+          name: hadoLint_openjdk_17
           command: |
-            docker run --rm -i hadolint/hadolint < docker/openjdk-11/Dockerfile
+            docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
       - run:
-          name: hadoLint_openjdk_11_debug
+          name: hadoLint_openjdk_17_debug
           command: |
-            docker run --rm -i hadolint/hadolint < docker/openjdk-11-debug/Dockerfile
+            docker run --rm -i hadolint/hadolint < docker/openjdk-17-debug/Dockerfile
       - run:
           name: hadoLint_openjdk_latest
           command: |
@@ -302,9 +303,9 @@ jobs:
       - attach_workspace:
           at: ~/project
       - run:
-          name: hadoLint_openjdk_11
+          name: hadoLint_openjdk_17
           command: |
-            docker run --rm -i hadolint/hadolint < docker/openjdk-11/Dockerfile
+            docker run --rm -i hadolint/hadolint < docker/openjdk-17/Dockerfile
       - run:
           name: hadoLint_openjdk_latest
           command: |
@@ -313,6 +314,11 @@ jobs:
           name: hadoLint_graalvm
           command: |
             docker run --rm -i hadolint/hadolint < docker/graalvm/Dockerfile
+      - run:
+          name: Java_17
+          command: |
+            sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless
+            sudo update-java-alternatives -a
       - run:
           name: build image
           command: |
@@ -354,6 +360,11 @@ jobs:
       - prepare
       - attach_workspace:
           at: ~/project
+      - run:
+          name: Java_17
+          command: |
+            sudo apt install -q --assume-yes openjdk-17-jre-headless openjdk-17-jdk-headless
+            sudo update-java-alternatives -a
       - run:
           name: Publish Docker
           command: |

.github/workflows/checks.yml

@@ -16,7 +16,7 @@ jobs:
         uses: actions/setup-java@v2
         with:
           distribution: adopt
-          java-version: 11
+          java-version: 17
           cache: gradle
       - name: spotless
         run: ./gradlew --no-daemon --parallel clean spotlessCheck

.github/workflows/codeql.yml

@@ -41,6 +41,12 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v3
 
+    - name: Set up Java
+      uses: actions/setup-java@v2
+      with:
+        distribution: adopt
+        java-version: 17
+
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v2

CHANGELOG.md

@@ -4,16 +4,20 @@
 
 ### Breaking Changes
 - Add a new CLI option to limit the number of requests in a single RPC batch request. Default=1 [#4965](https://github.com/hyperledger/besu/pull/4965)
-
 - Changed JsonRpc http service to return the error -32602 (Invalid params) with a 200 http status code
+- Besu requires minimum Java 17 and up to build and run [#3320](https://github.com/hyperledger/besu/issues/3320)
+- PKCS11 with nss module (PKCS11 based HSM can be used in DevP2P TLS and QBFT PKI) does not work with RSA keys 
+in Java 17. SoftHSM is tested manually and working. (Other PKCS11 HSM are not tested). The relevant unit and acceptance 
+tests are updated to use EC private keys instead of RSA keys.
 
 ### Additions and Improvements
 - Added option to evm CLI tool to allow code execution at specific forks [#4913](https://github.com/hyperledger/besu/pull/4913)
 - Improve get account performance by using the world state updater cache [#4897](https://github.com/hyperledger/besu/pull/4897)
 - Add new KZG precompile and option to override the trusted setup being used [#4822](https://github.com/hyperledger/besu/issues/4822)
 - Add implementation for eth_createAccessList RPC method [#4942](https://github.com/hyperledger/besu/pull/4942)
 - Updated reference tests to v11.3 [#4996](https://github.com/hyperledger/besu/pull/4996)
-- Add DebugGetRawBlock and DebugGetRawHeader RPC methods [#5011](https://github.com/hyperledger/besu/pull/5011)  
+- Add DebugGetRawBlock and DebugGetRawHeader RPC methods [#5011](https://github.com/hyperledger/besu/pull/5011)
+- Besu requires minimum Java 17 and up to build and run [#3320](https://github.com/hyperledger/besu/issues/3320)
 
 ### Bug Fixes
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java

@@ -16,6 +16,9 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static java.util.Collections.singletonList;
+import static org.hyperledger.besu.pki.keystore.KeyStoreWrapper.KEYSTORE_TYPE_JKS;
+import static org.hyperledger.besu.pki.keystore.KeyStoreWrapper.KEYSTORE_TYPE_PKCS11;
+import static org.hyperledger.besu.pki.keystore.KeyStoreWrapper.KEYSTORE_TYPE_PKCS12;
 
 import org.hyperledger.besu.cli.config.NetworkName;
 import org.hyperledger.besu.crypto.KeyPair;
@@ -34,7 +37,6 @@
 import org.hyperledger.besu.ethereum.permissioning.PermissioningConfiguration;
 import org.hyperledger.besu.metrics.prometheus.MetricsConfiguration;
 import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration;
-import org.hyperledger.besu.pki.keystore.KeyStoreWrapper;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationProvider;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.pki.PKCS11Utils;
 
@@ -366,38 +368,38 @@ public BesuNodeConfigurationBuilder p2pTLSEnabled(final String name, final Strin
     final TLSConfiguration.Builder builder = TLSConfiguration.Builder.tlsConfiguration();
     try {
       final String nsspin = "/pki-certs/%s/nsspin.txt";
-      final String truststore = "/pki-certs/%s/truststore.jks";
-      final String crl = "/pki-certs/%s/crl.pem";
+      final String truststore = "/pki-certs/%s/truststore.p12";
+      final String crl = "/pki-certs/crl/crl.pem";
       switch (type) {
-        case KeyStoreWrapper.KEYSTORE_TYPE_JKS:
+        case KEYSTORE_TYPE_JKS:
           builder
               .withKeyStoreType(type)
-              .withKeyStorePath(toPath(String.format("/pki-certs/%s/keystore.jks", name)))
+              .withKeyStorePath(toPath(String.format("/pki-certs/%s/%<s.jks", name)))
               .withKeyStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withKeyStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withTrustStoreType(type)
+              .withTrustStoreType(KEYSTORE_TYPE_PKCS12)
               .withTrustStorePath(toPath(String.format(truststore, name)))
               .withTrustStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withTrustStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withCrlPath(toPath(String.format(crl, name)));
+              .withCrlPath(toPath(crl));
           break;
-        case KeyStoreWrapper.KEYSTORE_TYPE_PKCS12:
+        case KEYSTORE_TYPE_PKCS12:
           builder
               .withKeyStoreType(type)
-              .withKeyStorePath(toPath(String.format("/pki-certs/%s/keys.p12", name)))
+              .withKeyStorePath(toPath(String.format("/pki-certs/%s/%<s.p12", name)))
               .withKeyStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withKeyStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withTrustStoreType(KeyStoreWrapper.KEYSTORE_TYPE_JKS)
+              .withTrustStoreType(KEYSTORE_TYPE_PKCS12)
               .withTrustStorePath(toPath(String.format(truststore, name)))
               .withTrustStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withTrustStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withCrlPath(toPath(String.format(crl, name)));
+              .withCrlPath(toPath(crl));
           break;
-        case KeyStoreWrapper.KEYSTORE_TYPE_PKCS11:
+        case KEYSTORE_TYPE_PKCS11:
           builder
               .withKeyStoreType(type)
               .withKeyStorePath(
@@ -406,7 +408,7 @@ public BesuNodeConfigurationBuilder p2pTLSEnabled(final String name, final Strin
               .withKeyStorePasswordSupplier(
                   new FileBasedPasswordProvider(toPath(String.format(nsspin, name))))
               .withKeyStorePasswordPath(toPath(String.format(nsspin, name)))
-              .withCrlPath(toPath(String.format(crl, name)));
+              .withCrlPath(toPath(crl));
           break;
       }
     } catch (Exception e) {

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/pki/PKCS11Utils.java

@@ -16,29 +16,36 @@
 package org.hyperledger.besu.tests.acceptance.dsl.node.configuration.pki;
 
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
-import java.nio.file.Files;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.nio.file.Path;
-
-import com.google.common.base.Charsets;
+import java.util.Properties;
 
 public class PKCS11Utils {
 
   public static Path initNSSConfigFile(final Path srcFilePath) {
-    Path ret = null;
-    try {
-      final String content = Files.readString(srcFilePath);
-      final String updated =
-          content.replaceAll(
-              "(nssSecmodDirectory\\W*)(\\.\\/.*)",
-              "$1".concat(srcFilePath.toAbsolutePath().toString().replace("nss.cfg", "nssdb")));
-      final Path targetFilePath = createTemporaryFile("nsscfg");
-      Files.write(targetFilePath, updated.getBytes(Charsets.UTF_8));
-      ret = targetFilePath;
-    } catch (IOException e) {
-      throw new RuntimeException("Error populating nss config file", e);
+    // load nss file as Properties
+    final Properties nssProp = new Properties();
+    try (InputStream input = new FileInputStream(srcFilePath.toFile())) {
+      nssProp.load(input);
+      String nssDbPath = srcFilePath.getParent().resolve("nssdb").toAbsolutePath().toString();
+      nssProp.setProperty("nssSecmodDirectory", nssDbPath);
+    } catch (final IOException e) {
+      throw new UncheckedIOException(e);
     }
-    return ret;
+
+    // store modified config into temporary file
+    final Path targetFilePath = createTemporaryFile("nsscfg");
+    try (FileOutputStream outputStream = new FileOutputStream(targetFilePath.toFile())) {
+      nssProp.store(outputStream, null);
+    } catch (final IOException e) {
+      throw new UncheckedIOException(e);
+    }
+
+    return targetFilePath;
   }
 
   private static Path createTemporaryFile(final String suffix) {

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/pki/PkiKeystoreConfigurationFactory.java

@@ -24,6 +24,7 @@
 
 import org.hyperledger.besu.pki.config.PkiKeyStoreConfiguration;
 import org.hyperledger.besu.pki.keystore.KeyStoreWrapper;
+import org.hyperledger.besu.pki.util.TestCertificateUtils;
 
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -46,8 +47,8 @@ public class PkiKeystoreConfigurationFactory {
   */
   final String NSSCONFIG_PATH_STRING = "/pki-certs/%s/nss.cfg";
   final String NSSPIN_PATH_STRING = "/pki-certs/%s/nsspin.txt";
-  final String TRUSTSTORE_PATH_STRING = "/pki-certs/%s/truststore.jks";
-  final String CRL_PATH_STRING = "/pki-certs/%s/crl.pem";
+  final String TRUSTSTORE_PATH_STRING = "/pki-certs/%s/truststore.p12";
+  final String CRL_PATH_STRING = "/pki-certs/crl/crl.pem";
 
   /*
    Software keystore config
@@ -78,10 +79,10 @@ private PkiKeyStoreConfiguration createPKCS11PkiConfig(final String name) {
               PKCS11Utils.initNSSConfigFile(
                   readResourceAsPath(String.format(NSSCONFIG_PATH_STRING, name))))
           .withKeyStorePasswordPath(readResourceAsPath(String.format(NSSPIN_PATH_STRING, name)))
-          .withTrustStoreType(KeyStoreWrapper.KEYSTORE_TYPE_JKS)
+          .withTrustStoreType(KeyStoreWrapper.KEYSTORE_TYPE_PKCS12)
           .withTrustStorePath(readResourceAsPath(String.format(TRUSTSTORE_PATH_STRING, name)))
           .withTrustStorePasswordPath(readResourceAsPath(String.format(NSSPIN_PATH_STRING, name)))
-          .withCrlFilePath(readResourceAsPath(String.format(CRL_PATH_STRING, name)))
+          .withCrlFilePath(readResourceAsPath(CRL_PATH_STRING))
           .withCertificateAlias(name);
 
     } catch (Exception e) {
@@ -113,7 +114,7 @@ private Path createTrustStore(final String type) {
     // Only create the truststore if this is the first time this method is being called
     if (caKeyPair == null) {
       try {
-        caKeyPair = createKeyPair();
+        caKeyPair = createKeyPair(TestCertificateUtils.Algorithm.RSA);
         caCertificate = createSelfSignedCertificate("ca", notBefore(), notAfter(), caKeyPair);
 
         final KeyStore truststore = KeyStore.getInstance(type);
@@ -136,7 +137,7 @@ private Path createKeyStore(final String type, final String alias) {
       createTrustStore(type);
     }
 
-    final KeyPair kp = createKeyPair();
+    final KeyPair kp = createKeyPair(TestCertificateUtils.Algorithm.RSA);
     final X509Certificate certificate =
         issueCertificate(caCertificate, caKeyPair, "validator", notBefore(), notAfter(), kp, false);