Merge pull request #1 from codyborn/merge1153

Merge1153

.circleci/config.yml

@@ -262,7 +262,7 @@ jobs:
           docker_layer_caching: true
       - run:
           name: Quorum Acceptance Tests
-          no_output_timeout: 30m
+          no_output_timeout: 5m
           command: ./gradlew --no-daemon acceptanceTestsQuorum
       - store_artifacts:
           path: build/quorum-at

MAINTAINERS.md

@@ -8,15 +8,16 @@
 
 | Name             | Github           | LFID             |
 | ---------------- | ---------------- | ---------------- |
+| Ameziane Hamlat   | ahamlat         | ahamlat          |
 | Adrian Sutton    | ajsutton         | ajsutton         |
 | Antony Denyer    | antonydenyer     | antonydenyer     |
 | Antoine Toulme   | atoulme          | atoulme          |
 | Byron Gravenorst | bgravenorst      | bgravenorst      |
 | Daniel Lehrner   | daniellehrner    | daniellehrner    |
 | Diego López León | diega            | diega            |
 | Fabio Di Fabio   | fab-10           | fab-10           |
-| Frank Li         | frankisawesome   | frankliawesome   |
 | Gary Schulte     | garyschulte      | GarySchulte      | 
+| Gabriel Trintinalia | gabriel-trintinalia | gabrieltrintinalia |
 | Jiri Peinlich    | gezero           | JiriPeinlich     |
 | Justin Florentine| jflo             | RoboCopsGoneMad  |
 | Jason Frame      | jframe           | jframe           |
@@ -42,6 +43,7 @@
 | David Mechler    | davemec          | davemec          |
 | Edward Evans     | EdJoJob          | EdJoJob          |
 | Edward Mack      | edwardmack       | mackcom          | 
+| Frank Li         | frankisawesome   | frankliawesome   |
 | Ivaylo Kirilov   | iikirilov        | iikirilov        |
 | Madeline Murray  | MadelineMurray   | madelinemurray   |
 | Trent Mohay      | rain-on          | trent.mohay      |

SECURITY.md

@@ -7,13 +7,15 @@ hear from you. We will take all security bugs seriously and if confirmed upon in
 patch it within a reasonable amount of time and release a public security bulletin discussing the
 impact and credit the discoverer.
 
-There are two ways to report a security bug. The easiest is to email a description of the flaw and
-any related information (e.g. reproduction steps, version) to
-[security at hyperledger dot org](mailto:security@hyperledger.org).
-
-The other way is to file a confidential security bug in our
-[JIRA bug tracking system](https://jira.hyperledger.org). Be sure to set the “Security Level” to
-“Security issue”.
+There are two email addresses where Hyperledger Besu accepts security bugs. The
+first, [security "dash" besu at lists dot hyperledger dot org](mailto:security-besu@lists.hyperledger.org)
+is limited to a subset of Hyperledger Besu maintainers and Hyperledger staff. For highly sensitive
+bugs this is a preferred address. The second email
+address [security at hyperledger dot org](mailto:security@hyperledger.org) is limited to a subset of
+maintainers and staff of all Hyperledger projects, and may be viewed by maintainers outside of
+Hyperledger Besu. When sending information to either of these emails please be sure to include a
+description of the flaw and any related information (e.g. reproduction steps, version, known active
+use).
 
 The process by which the Hyperledger Security Team handles security bugs is documented further in
 our [Defect Response page](https://wiki.hyperledger.org/display/SEC/Defect+Response) on our

acceptance-tests/dsl/build.gradle

@@ -33,6 +33,7 @@ dependencies {
   implementation 'io.reactivex.rxjava2:rxjava'
   implementation 'io.vertx:vertx-core'
   implementation 'junit:junit'
+  implementation 'io.opentelemetry:opentelemetry-api'
   implementation 'org.apache.tuweni:tuweni-bytes'
   implementation 'org.apache.tuweni:tuweni-io'
   implementation 'org.apache.tuweni:tuweni-units'

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/BesuNode.java

@@ -127,6 +127,7 @@ public class BesuNode implements NodeConfiguration, RunnableNode, AutoCloseable
   private Optional<Integer> exitCode = Optional.empty();
   private Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration = Optional.empty();
   private final boolean isStrictTxReplayProtectionEnabled;
+  private final Map<String, String> environment;
 
   public BesuNode(
       final String name,
@@ -159,7 +160,8 @@ public BesuNode(
       final List<String> runCommand,
       final Optional<KeyPair> keyPair,
       final Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration,
-      final boolean isStrictTxReplayProtectionEnabled)
+      final boolean isStrictTxReplayProtectionEnabled,
+      final Map<String, String> environment)
       throws IOException {
     this.homeDirectory = dataPath.orElseGet(BesuNode::createTmpDataDirectory);
     this.isStrictTxReplayProtectionEnabled = isStrictTxReplayProtectionEnabled;
@@ -216,6 +218,7 @@ public BesuNode(
     this.isDnsEnabled = isDnsEnabled;
     privacyParameters.ifPresent(this::setPrivacyParameters);
     this.pkiKeyStoreConfiguration = pkiKeyStoreConfiguration;
+    this.environment = environment;
     LOG.info("Created BesuNode {}", this);
   }
 
@@ -298,7 +301,7 @@ private String getDiscoveryPort() {
     return port;
   }
 
-  private Optional<String> jsonRpcBaseUrl() {
+  public Optional<String> jsonRpcBaseUrl() {
     if (isJsonRpcEnabled()) {
       return Optional.of(
           HTTP + jsonRpcConfiguration.getHost() + ":" + portsProperties.getProperty(JSON_RPC));
@@ -794,4 +797,9 @@ public void verify(final Condition expected) {
   public void setExitCode(final int exitValue) {
     this.exitCode = Optional.of(exitValue);
   }
+
+  @Override
+  public Map<String, String> getEnvironment() {
+    return environment;
+  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -419,6 +419,8 @@ public void startNode(final BesuNode node) {
             "JAVA_OPTS",
             "-Djava.security.properties="
                 + "acceptance-tests/tests/build/resources/test/acceptanceTesting.security");
+    // add additional environment variables
+    processBuilder.environment().putAll(node.getEnvironment());
     try {
       checkState(
           isNotAliveOrphan(node.getName()),

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -65,6 +65,7 @@
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 
+import io.opentelemetry.api.GlobalOpenTelemetry;
 import io.vertx.core.Vertx;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -133,6 +134,7 @@ public void startNode(final BesuNode node) {
                 buildPluginContext(
                     node, storageService, securityModuleService, commonPluginConfiguration));
 
+    GlobalOpenTelemetry.resetForTest();
     final ObservableMetricsSystem metricsSystem =
         MetricsSystemFactory.create(node.getMetricsConfiguration());
     final List<EnodeURL> bootnodes =
@@ -218,7 +220,6 @@ public void startNode(final BesuNode node) {
         .besuPluginContext(new BesuPluginContextImpl())
         .autoLogBloomCaching(false)
         .storageProvider(storageProvider)
-        .forkIdSupplier(() -> besuController.getProtocolManager().getForkIdAsBytesList())
         .rpcEndpointService(new RpcEndpointServiceImpl());
     node.engineRpcConfiguration().ifPresent(runnerBuilder::engineJsonRpcConfiguration);
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfiguration.java

@@ -30,6 +30,7 @@
 
 import java.nio.file.Path;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 public class BesuNodeConfiguration {
@@ -65,6 +66,7 @@ public class BesuNodeConfiguration {
   private final Optional<KeyPair> keyPair;
   private final Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration;
   private final boolean strictTxReplayProtectionEnabled;
+  private final Map<String, String> environment;
 
   BesuNodeConfiguration(
       final String name,
@@ -97,7 +99,8 @@ public class BesuNodeConfiguration {
       final List<String> runCommand,
       final Optional<KeyPair> keyPair,
       final Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration,
-      final boolean strictTxReplayProtectionEnabled) {
+      final boolean strictTxReplayProtectionEnabled,
+      final Map<String, String> environment) {
     this.name = name;
     this.miningParameters = miningParameters;
     this.jsonRpcConfiguration = jsonRpcConfiguration;
@@ -129,6 +132,7 @@ public class BesuNodeConfiguration {
     this.keyPair = keyPair;
     this.pkiKeyStoreConfiguration = pkiKeyStoreConfiguration;
     this.strictTxReplayProtectionEnabled = strictTxReplayProtectionEnabled;
+    this.environment = environment;
   }
 
   public String getName() {
@@ -254,4 +258,8 @@ public Optional<PkiKeyStoreConfiguration> getPkiKeyStoreConfiguration() {
   public boolean isStrictTxReplayProtectionEnabled() {
     return strictTxReplayProtectionEnabled;
   }
+
+  public Map<String, String> getEnvironment() {
+    return environment;
+  }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeConfigurationBuilder.java

@@ -44,7 +44,9 @@
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 public class BesuNodeConfigurationBuilder {
@@ -85,6 +87,7 @@ public class BesuNodeConfigurationBuilder {
   private Optional<KeyPair> keyPair = Optional.empty();
   private Optional<PkiKeyStoreConfiguration> pkiKeyStoreConfiguration = Optional.empty();
   private Boolean strictTxReplayProtectionEnabled = false;
+  private Map<String, String> environment = new HashMap<>();
 
   public BesuNodeConfigurationBuilder() {
     // Check connections more frequently during acceptance tests to cut down on
@@ -179,6 +182,11 @@ public BesuNodeConfigurationBuilder jsonRpcAdmin() {
     return this;
   }
 
+  public BesuNodeConfigurationBuilder jsonRpcDebug() {
+    this.jsonRpcConfiguration.addRpcApi(RpcApis.DEBUG.name());
+    return this;
+  }
+
   public BesuNodeConfigurationBuilder jsonRpcAuthenticationConfiguration(final String authFile)
       throws URISyntaxException {
     final String authTomlPath =
@@ -478,6 +486,11 @@ public BesuNodeConfigurationBuilder strictTxReplayProtectionEnabled(
     return this;
   }
 
+  public BesuNodeConfigurationBuilder environment(final Map<String, String> environment) {
+    this.environment = environment;
+    return this;
+  }
+
   public BesuNodeConfiguration build() {
     return new BesuNodeConfiguration(
         name,
@@ -510,6 +523,7 @@ public BesuNodeConfiguration build() {
         runCommand,
         keyPair,
         pkiKeyStoreConfiguration,
-        strictTxReplayProtectionEnabled);
+        strictTxReplayProtectionEnabled,
+        environment);
   }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/BesuNodeFactory.java

@@ -88,7 +88,8 @@ public BesuNode create(final BesuNodeConfiguration config) throws IOException {
         config.getRunCommand(),
         config.getKeyPair(),
         config.getPkiKeyStoreConfiguration(),
-        config.isStrictTxReplayProtectionEnabled());
+        config.isStrictTxReplayProtectionEnabled(),
+        config.getEnvironment());
   }
 
   public BesuNode createMinerNode(
@@ -512,6 +513,7 @@ public BesuNode createCustomGenesisNode(
   public BesuNode createExecutionEngineGenesisNode(final String name, final String genesisPath)
       throws IOException {
     final String genesisFile = GenesisConfigurationFactory.readGenesisFile(genesisPath);
+
     return create(
         new BesuNodeConfigurationBuilder()
             .name(name)
@@ -521,6 +523,7 @@ public BesuNode createExecutionEngineGenesisNode(final String name, final String
             .miningEnabled()
             .jsonRpcEnabled()
             .engineRpcEnabled(true)
+            .jsonRpcDebug()
             .build());
   }
 

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/NodeConfiguration.java

@@ -18,6 +18,7 @@
 
 import java.net.URI;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 
 public interface NodeConfiguration {
@@ -59,4 +60,6 @@ public interface NodeConfiguration {
   boolean isRevertReasonEnabled();
 
   List<String> getStaticNodes();
+
+  Map<String, String> getEnvironment();
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/privacy/PrivacyNodeFactory.java

@@ -19,6 +19,7 @@
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationFactory;
 import org.hyperledger.besu.tests.acceptance.dsl.privacy.PrivacyNode;
 import org.hyperledger.besu.tests.acceptance.dsl.privacy.account.PrivacyAccount;
+import org.hyperledger.enclave.testutil.EnclaveEncryptorType;
 import org.hyperledger.enclave.testutil.EnclaveKeyConfiguration;
 import org.hyperledger.enclave.testutil.EnclaveType;
 
@@ -233,8 +234,11 @@ public PrivacyNode createFlexiblePrivacyGroupEnabledMinerNode(
     final BesuNodeConfigurationBuilder besuNodeConfigurationBuilder =
         new BesuNodeConfigurationBuilder();
     if (multiTenancyEnabled) {
-      besuNodeConfigurationBuilder.jsonRpcAuthenticationConfiguration(
-          "authentication/auth_priv.toml");
+      final String authPrivTomlPath =
+          EnclaveEncryptorType.EC.equals(privacyAccount.getEnclaveEncryptorType())
+              ? "authentication/auth_priv_ec_pubkey.toml"
+              : "authentication/auth_priv.toml";
+      besuNodeConfigurationBuilder.jsonRpcAuthenticationConfiguration(authPrivTomlPath);
     }
     return create(
         new PrivacyNodeConfiguration(
@@ -252,7 +256,9 @@ public PrivacyNode createFlexiblePrivacyGroupEnabledMinerNode(
                 .keyFilePath(privacyAccount.getPrivateKeyPath())
                 .build(),
             new EnclaveKeyConfiguration(
-                privacyAccount.getEnclaveKeyPaths(), privacyAccount.getEnclavePrivateKeyPaths())),
+                privacyAccount.getEnclaveKeyPaths(),
+                privacyAccount.getEnclavePrivateKeyPaths(),
+                privacyAccount.getEnclaveEncryptorType())),
         enclaveType,
         containerNetwork);
   }
@@ -279,7 +285,9 @@ public PrivacyNode createFlexiblePrivacyGroupEnabledNode(
                 .webSocketEnabled()
                 .build(),
             new EnclaveKeyConfiguration(
-                privacyAccount.getEnclaveKeyPaths(), privacyAccount.getEnclavePrivateKeyPaths())),
+                privacyAccount.getEnclaveKeyPaths(),
+                privacyAccount.getEnclavePrivateKeyPaths(),
+                privacyAccount.getEnclaveEncryptorType())),
         enclaveType,
         containerNetwork);
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java

@@ -15,6 +15,11 @@
 package org.hyperledger.besu.tests.acceptance.dsl.privacy;
 
 import static org.hyperledger.besu.controller.BesuController.DATABASE_PATH;
+import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_BACKGROUND_THREAD_COUNT;
+import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_CACHE_CAPACITY;
+import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_IS_HIGH_SPEC;
+import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_BACKGROUND_COMPACTIONS;
+import static org.hyperledger.besu.plugin.services.storage.rocksdb.configuration.RocksDBCLIOptions.DEFAULT_MAX_OPEN_FILES;
 
 import org.hyperledger.besu.crypto.KeyPairUtil;
 import org.hyperledger.besu.datatypes.Address;
@@ -65,10 +70,6 @@
 public class PrivacyNode implements AutoCloseable {
 
   private static final Logger LOG = LoggerFactory.getLogger(PrivacyNode.class);
-  private static final int MAX_OPEN_FILES = 1024;
-  private static final long CACHE_CAPACITY = 8388608;
-  private static final int MAX_BACKGROUND_COMPACTIONS = 4;
-  private static final int BACKGROUND_THREAD_COUNT = 4;
 
   private final EnclaveTestHarness enclave;
   private final BesuNode besu;
@@ -127,7 +128,8 @@ public PrivacyNode(
             List.of(),
             Optional.empty(),
             Optional.empty(),
-            besuConfig.isStrictTxReplayProtectionEnabled());
+            besuConfig.isStrictTxReplayProtectionEnabled(),
+            besuConfig.getEnvironment());
   }
 
   public void testEnclaveConnection(final List<PrivacyNode> otherNodes) {
@@ -275,10 +277,11 @@ private PrivacyStorageProvider createKeyValueStorageProvider(
                 new RocksDBKeyValueStorageFactory(
                     () ->
                         new RocksDBFactoryConfiguration(
-                            MAX_OPEN_FILES,
-                            MAX_BACKGROUND_COMPACTIONS,
-                            BACKGROUND_THREAD_COUNT,
-                            CACHE_CAPACITY),
+                            DEFAULT_MAX_OPEN_FILES,
+                            DEFAULT_MAX_BACKGROUND_COMPACTIONS,
+                            DEFAULT_BACKGROUND_THREAD_COUNT,
+                            DEFAULT_CACHE_CAPACITY,
+                            DEFAULT_IS_HIGH_SPEC),
                     Arrays.asList(KeyValueSegmentIdentifier.values()),
                     RocksDBMetricsFactory.PRIVATE_ROCKS_DB_METRICS)))
         .withCommonConfiguration(new BesuConfigurationImpl(dataLocation, dbLocation))