Adds unrestricted privacy (hyperledger#2324)

* Unrestricted Privacy: implement unrestricted privacy

- put data on chain
- wire up methods for sendRaw with restriction
- unrestricted pre-compile for processing transactions at Address.precompiled(PRIVACY - 4)
- store private state of unrestricted transactions
- route priv endpoints to unrestricted state resolution
- Implement unrestricted websocket endpoint
- Tidy up web3j transaction manager naming
- Parameterize tests for different privacy restrictions
- Implement our own PrivateTransactionManager
- remove chainId from sendRawTransaction calls
- Add check for member being a participant when creating privacy group
- refactor private marker transaction naming
- mark privacy-unrestricted-enabled as beta
- Remove create privacy group from unrestricted
- Unrestricted privacy acceptance tests will use a generated group id.
- rename enclavePublicKey to privacyUserId
- Ignore some tests for unrestricted privacy
- privacyGroupId has no significant meaning in unrestricted tests
- Change config label to be inline with previous conventions
- command tests to be added when made stable
Signed-off-by: Antony Denyer <git@antonydenyer.co.uk>
Signed-off-by: Stefan Pingel <stefan.pingel@consensys.net>
Co-authored-by: Stefan Pingel <stefan.pingel@consensys.net>

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ProcessBesuNodeRunner.java

@@ -120,20 +120,27 @@ public void startNode(final BesuNode node) {
 
     if (node.getPrivacyParameters().isEnabled()) {
       params.add("--privacy-enabled");
+
       params.add("--privacy-url");
       params.add(node.getPrivacyParameters().getEnclaveUri().toString());
+
       if (node.getPrivacyParameters().isMultiTenancyEnabled()) {
         params.add("--privacy-multi-tenancy-enabled");
       } else {
         params.add("--privacy-public-key-file");
         params.add(node.getPrivacyParameters().getEnclavePublicKeyFile().getAbsolutePath());
       }
+
       params.add("--privacy-marker-transaction-signing-key-file");
       params.add(node.homeDirectory().resolve("key").toString());
 
       if (node.getPrivacyParameters().isOnchainPrivacyGroupsEnabled()) {
         params.add("--privacy-onchain-groups-enabled");
       }
+
+      if (node.getPrivacyParameters().isUnrestrictedPrivacyEnabled()) {
+        params.add("--Xprivacy-unrestricted-enabled");
+      }
     }
 
     params.add("--bootnodes");

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/privacy/PrivacyNodeConfiguration.java

@@ -19,34 +19,28 @@
 
 public class PrivacyNodeConfiguration {
 
-  private final int privacyAddress;
   private final boolean isOnchainPrivacyGroupEnabled;
+  private final boolean isMultitenancyEnabled;
+  private final boolean isUnrestrictedEnabled;
   private final BesuNodeConfiguration besuConfig;
   private final EnclaveKeyConfiguration keyConfig;
-  private final boolean isMultitenancyEnabled;
 
   PrivacyNodeConfiguration(
-      final int privacyAddress,
-      final BesuNodeConfiguration besuConfig,
-      final EnclaveKeyConfiguration keyConfig) {
-    this(privacyAddress, false, false, besuConfig, keyConfig);
+      final BesuNodeConfiguration besuConfig, final EnclaveKeyConfiguration keyConfig) {
+    this(false, false, false, besuConfig, keyConfig);
   }
 
   PrivacyNodeConfiguration(
-      final int privacyAddress,
       final boolean isOnchainPrivacyGroupEnabled,
       final boolean isMultitenancyEnabled,
+      final boolean isUnrestrictedEnabled,
       final BesuNodeConfiguration besuConfig,
       final EnclaveKeyConfiguration keyConfig) {
-    this.privacyAddress = privacyAddress;
     this.isOnchainPrivacyGroupEnabled = isOnchainPrivacyGroupEnabled;
     this.besuConfig = besuConfig;
     this.keyConfig = keyConfig;
     this.isMultitenancyEnabled = isMultitenancyEnabled;
-  }
-
-  public int getPrivacyAddress() {
-    return privacyAddress;
+    this.isUnrestrictedEnabled = isUnrestrictedEnabled;
   }
 
   public boolean isOnchainPrivacyGroupEnabled() {
@@ -57,6 +51,10 @@ public boolean isMultitenancyEnabled() {
     return isMultitenancyEnabled;
   }
 
+  public boolean isUnrestrictedEnabled() {
+    return isUnrestrictedEnabled;
+  }
+
   public BesuNodeConfiguration getBesuConfig() {
     return besuConfig;
   }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/configuration/privacy/PrivacyNodeFactory.java

@@ -14,7 +14,6 @@
  */
 package org.hyperledger.besu.tests.acceptance.dsl.node.configuration.privacy;
 
-import org.hyperledger.besu.ethereum.core.Address;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.BesuNodeConfigurationBuilder;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.NodeConfigurationFactory;
 import org.hyperledger.besu.tests.acceptance.dsl.node.configuration.genesis.GenesisConfigurationFactory;
@@ -55,19 +54,23 @@ public PrivacyNode createPrivateTransactionEnabledMinerNode(
       final Optional<Network> containerNetwork)
       throws IOException {
     return createPrivateTransactionEnabledMinerNode(
-        name, privacyAccount, Address.PRIVACY, enclaveType, containerNetwork);
+        name, privacyAccount, enclaveType, containerNetwork, false, false, false);
   }
 
   public PrivacyNode createPrivateTransactionEnabledMinerNode(
       final String name,
       final PrivacyAccount privacyAccount,
-      final int privacyAddress,
       final EnclaveType enclaveType,
-      final Optional<Network> containerNetwork)
+      final Optional<Network> containerNetwork,
+      final boolean isOnchainPrivacyGroupEnabled,
+      final boolean isMultitenancyEnabled,
+      final boolean isUnrestrictedEnabled)
       throws IOException {
     return create(
         new PrivacyNodeConfiguration(
-            privacyAddress,
+            isOnchainPrivacyGroupEnabled,
+            isMultitenancyEnabled,
+            isUnrestrictedEnabled,
             new BesuNodeConfigurationBuilder()
                 .name(name)
                 .miningEnabled()
@@ -89,19 +92,23 @@ public PrivacyNode createPrivateTransactionEnabledNode(
       final Optional<Network> containerNetwork)
       throws IOException {
     return createPrivateTransactionEnabledNode(
-        name, privacyAccount, Address.PRIVACY, enclaveType, containerNetwork);
+        name, privacyAccount, enclaveType, containerNetwork, false, false, false);
   }
 
   public PrivacyNode createPrivateTransactionEnabledNode(
       final String name,
       final PrivacyAccount privacyAccount,
-      final int privacyAddress,
       final EnclaveType enclaveType,
-      final Optional<Network> containerNetwork)
+      final Optional<Network> containerNetwork,
+      final boolean isOnchainPrivacyGroupEnabled,
+      final boolean isMultitenancyEnabled,
+      final boolean isUnrestrictedEnabled)
       throws IOException {
     return create(
         new PrivacyNodeConfiguration(
-            privacyAddress,
+            isOnchainPrivacyGroupEnabled,
+            isMultitenancyEnabled,
+            isUnrestrictedEnabled,
             new BesuNodeConfigurationBuilder()
                 .name(name)
                 .jsonRpcEnabled()
@@ -115,37 +122,31 @@ public PrivacyNode createPrivateTransactionEnabledNode(
         containerNetwork);
   }
 
-  public PrivacyNode createIbft2NodePrivacyMiningEnabled(
-      final String name,
-      final PrivacyAccount privacyAccount,
-      final EnclaveType enclaveType,
-      final Optional<Network> containerNetwork)
-      throws IOException {
-    return createIbft2NodePrivacyEnabled(
-        name, privacyAccount, Address.PRIVACY, true, enclaveType, containerNetwork);
-  }
-
   public PrivacyNode createIbft2NodePrivacyEnabled(
       final String name,
       final PrivacyAccount privacyAccount,
       final EnclaveType enclaveType,
       final Optional<Network> containerNetwork)
       throws IOException {
     return createIbft2NodePrivacyEnabled(
-        name, privacyAccount, Address.PRIVACY, false, enclaveType, containerNetwork);
+        name, privacyAccount, false, enclaveType, containerNetwork, false, false, false);
   }
 
   public PrivacyNode createIbft2NodePrivacyEnabled(
       final String name,
       final PrivacyAccount privacyAccount,
-      final int privacyAddress,
       final boolean minerEnabled,
       final EnclaveType enclaveType,
-      final Optional<Network> containerNetwork)
+      final Optional<Network> containerNetwork,
+      final boolean isOnchainPrivacyGroupEnabled,
+      final boolean isMultitenancyEnabled,
+      final boolean isUnrestrictedEnabled)
       throws IOException {
     return create(
         new PrivacyNodeConfiguration(
-            privacyAddress,
+            isOnchainPrivacyGroupEnabled,
+            isMultitenancyEnabled,
+            isUnrestrictedEnabled,
             new BesuNodeConfigurationBuilder()
                 .name(name)
                 .miningEnabled()
@@ -166,11 +167,16 @@ public PrivacyNode createQbftNodePrivacyEnabled(
       final String name,
       final PrivacyAccount privacyAccount,
       final EnclaveType enclaveType,
-      final Optional<Network> containerNetwork)
+      final Optional<Network> containerNetwork,
+      final boolean isOnchainPrivacyGroupEnabled,
+      final boolean isMultitenancyEnabled,
+      final boolean isUnrestrictedEnabled)
       throws IOException {
     return create(
         new PrivacyNodeConfiguration(
-            Address.PRIVACY,
+            isOnchainPrivacyGroupEnabled,
+            isMultitenancyEnabled,
+            isUnrestrictedEnabled,
             new BesuNodeConfigurationBuilder()
                 .name(name)
                 .miningEnabled()
@@ -190,7 +196,6 @@ public PrivacyNode createQbftNodePrivacyEnabled(
   public PrivacyNode createOnChainPrivacyGroupEnabledMinerNode(
       final String name,
       final PrivacyAccount privacyAccount,
-      final int privacyAddress,
       final boolean multiTenancyEnabled,
       final EnclaveType enclaveType,
       final Optional<Network> containerNetwork)
@@ -203,9 +208,9 @@ public PrivacyNode createOnChainPrivacyGroupEnabledMinerNode(
     }
     return create(
         new PrivacyNodeConfiguration(
-            privacyAddress,
             true,
             multiTenancyEnabled,
+            false,
             besuNodeConfigurationBuilder
                 .name(name)
                 .miningEnabled()
@@ -223,16 +228,15 @@ public PrivacyNode createOnChainPrivacyGroupEnabledMinerNode(
   public PrivacyNode createOnChainPrivacyGroupEnabledNode(
       final String name,
       final PrivacyAccount privacyAccount,
-      final int privacyAddress,
       final boolean multiTenancyEnabled,
       final EnclaveType enclaveType,
       final Optional<Network> containerNetwork)
       throws IOException {
     return create(
         new PrivacyNodeConfiguration(
-            privacyAddress,
             true,
             multiTenancyEnabled,
+            false,
             new BesuNodeConfigurationBuilder()
                 .name(name)
                 .jsonRpcEnabled()

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/ParameterizedEnclaveTestBase.java

@@ -14,24 +14,55 @@
  */
 package org.hyperledger.besu.tests.acceptance.dsl.privacy;
 
+import static org.hyperledger.enclave.testutil.EnclaveType.NOOP;
+import static org.hyperledger.enclave.testutil.EnclaveType.ORION;
+import static org.hyperledger.enclave.testutil.EnclaveType.TESSERA;
+import static org.web3j.utils.Restriction.RESTRICTED;
+import static org.web3j.utils.Restriction.UNRESTRICTED;
+
+import org.hyperledger.besu.tests.acceptance.dsl.privacy.transaction.RestrictedCreatePrivacyGroupTransaction;
+import org.hyperledger.besu.tests.acceptance.dsl.privacy.transaction.UnrestrictedCreateRandomPrivacyGroupIdTransaction;
+import org.hyperledger.besu.tests.acceptance.dsl.transaction.Transaction;
 import org.hyperledger.enclave.testutil.EnclaveType;
 
 import java.util.Arrays;
 import java.util.Collection;
 
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.web3j.utils.Restriction;
 
 @RunWith(Parameterized.class)
 public abstract class ParameterizedEnclaveTestBase extends PrivacyAcceptanceTestBase {
+  protected final Restriction restriction;
   protected final EnclaveType enclaveType;
 
-  protected ParameterizedEnclaveTestBase(final EnclaveType enclaveType) {
+  protected ParameterizedEnclaveTestBase(
+      final Restriction restriction, final EnclaveType enclaveType) {
+    this.restriction = restriction;
     this.enclaveType = enclaveType;
   }
 
-  @Parameterized.Parameters(name = "{0}")
-  public static Collection<EnclaveType> enclaveTypes() {
-    return Arrays.asList(EnclaveType.values());
+  @Parameters(name = "{0} tx with {1} enclave")
+  public static Collection<Object[]> params() {
+    return Arrays.asList(
+        new Object[][] {
+          {RESTRICTED, TESSERA},
+          {RESTRICTED, ORION},
+          {UNRESTRICTED, NOOP}
+        });
+  }
+
+  public Transaction<String> createPrivacyGroup(
+      final String name, final String description, final PrivacyNode... nodes) {
+
+    if (restriction == RESTRICTED) {
+      return new RestrictedCreatePrivacyGroupTransaction(name, description, nodes);
+    } else if (restriction == UNRESTRICTED) {
+      return new UnrestrictedCreateRandomPrivacyGroupIdTransaction();
+    } else {
+      throw new RuntimeException("Do not know how to handle " + restriction);
+    }
   }
 }

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/privacy/PrivacyNode.java

@@ -43,6 +43,7 @@
 import org.hyperledger.besu.tests.acceptance.dsl.transaction.Transaction;
 import org.hyperledger.enclave.testutil.EnclaveTestHarness;
 import org.hyperledger.enclave.testutil.EnclaveType;
+import org.hyperledger.enclave.testutil.NoopEnclaveTestHarness;
 import org.hyperledger.enclave.testutil.OrionTestHarnessFactory;
 import org.hyperledger.enclave.testutil.TesseraTestHarnessFactory;
 
@@ -75,6 +76,7 @@ public class PrivacyNode implements AutoCloseable {
   private final Vertx vertx;
   private final boolean isOnchainPrivacyEnabled;
   private final boolean isMultitenancyEnabled;
+  private final boolean isUnrestrictedEnabled;
 
   public PrivacyNode(
       final PrivacyNodeConfiguration privacyConfiguration,
@@ -92,6 +94,7 @@ public PrivacyNode(
 
     isOnchainPrivacyEnabled = privacyConfiguration.isOnchainPrivacyGroupEnabled();
     isMultitenancyEnabled = privacyConfiguration.isMultitenancyEnabled();
+    isUnrestrictedEnabled = privacyConfiguration.isUnrestrictedEnabled();
 
     this.besu =
         new BesuNode(
@@ -123,6 +126,11 @@ public PrivacyNode(
   }
 
   public void testEnclaveConnection(final List<PrivacyNode> otherNodes) {
+    if (this.isUnrestrictedEnabled) {
+      LOG.info("Skipping as node has no enclave (isUnrestrictedEnabled=true)");
+      return;
+    }
+
     if (!otherNodes.isEmpty()) {
       LOG.debug(
           String.format(
@@ -190,15 +198,16 @@ public void start(final BesuNodeRunner runner) {
           new PrivacyParameters.Builder()
               .setEnabled(true)
               .setEnclaveUrl(enclave.clientUrl())
-              .setEnclavePublicKeyUsingFile(enclave.getPublicKeyPaths().get(0).toFile())
+              .setPrivacyUserIdUsingFile(enclave.getPublicKeyPaths().get(0).toFile())
               .setStorageProvider(createKeyValueStorageProvider(dataDir, dbDir))
               .setPrivateKeyPath(KeyPairUtil.getDefaultKeyFile(besu.homeDirectory()).toPath())
               .setEnclaveFactory(new EnclaveFactory(vertx))
               .setOnchainPrivacyGroupsEnabled(isOnchainPrivacyEnabled)
               .setMultiTenancyEnabled(isMultitenancyEnabled)
+              .setUnrestrictedPrivacyEnabled(isUnrestrictedEnabled)
               .build();
     } catch (final IOException e) {
-      throw new RuntimeException();
+      throw new RuntimeException(e);
     }
     besu.setPrivacyParameters(privacyParameters);
     besu.start(runner);
@@ -273,10 +282,16 @@ private EnclaveTestHarness selectEnclave(
       final BesuNodeConfiguration config,
       final PrivacyNodeConfiguration privacyConfiguration,
       final Optional<Network> containerNetwork) {
-    return enclaveType == EnclaveType.TESSERA
-        ? TesseraTestHarnessFactory.create(
-            config.getName(), tempDir, privacyConfiguration.getKeyConfig(), containerNetwork)
-        : OrionTestHarnessFactory.create(
+
+    switch (enclaveType) {
+      case ORION:
+        return OrionTestHarnessFactory.create(
             config.getName(), tempDir, privacyConfiguration.getKeyConfig());
+      case TESSERA:
+        return TesseraTestHarnessFactory.create(
+            config.getName(), tempDir, privacyConfiguration.getKeyConfig(), containerNetwork);
+      default:
+        return new NoopEnclaveTestHarness(tempDir, privacyConfiguration.getKeyConfig());
+    }
   }
 }