[PRIV] Complete Private Transaction Processor (PegaSysEng#938)

* Remove entries() in KeyValueStorage

* Implement Private Transaction Processor

* Implement PrivateTransactionReceipt

* Refactor privacy parameter injection

* Disable ether for private transaction

* Implement AcceptanceTest

* Fix PrivateWorldState injection

* Add unimplemented private transaction type

- https://entethalliance.github.io/client-spec/spec.html#sec-extensions-json-rpc

* Change Private Transaction RLP

* Change Private Transaction RLP for acceptanceTest

* Fix copyright year

* Fix build

acceptance-tests/build.gradle

@@ -25,8 +25,10 @@ dependencies {
   testImplementation project(':ethereum:blockcreation')
   testImplementation project(':ethereum:jsonrpc')
   testImplementation project(':ethereum:permissioning')
+  testImplementation project(':ethereum:rlp')
   testImplementation project(':metrics')
   testImplementation project(':pantheon')
+  testImplementation project(':testutil')
   testImplementation project(':util')
   testImplementation project(path: ':ethereum:core', configuration: 'testSupportArtifacts')
 
@@ -35,6 +37,7 @@ dependencies {
   testImplementation 'io.reactivex.rxjava2:rxjava'
   testImplementation 'io.vertx:vertx-core'
   testImplementation 'junit:junit'
+  testImplementation 'net.consensys:orion'
   testImplementation 'org.apache.logging.log4j:log4j-api'
   testImplementation 'org.assertj:assertj-core'
   testImplementation 'org.awaitility:awaitility'

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/jsonrpc/Eth.java

@@ -12,7 +12,6 @@
  */
 package tech.pegasys.pantheon.tests.acceptance.dsl.jsonrpc;
 
-import tech.pegasys.pantheon.ethereum.core.Hash;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.Condition;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.eth.ExpectEthAccountsException;
 import tech.pegasys.pantheon.tests.acceptance.dsl.condition.eth.ExpectEthGetTransactionReceiptIsAbsent;
@@ -42,9 +41,9 @@ public Condition accountsExceptional(final String expectedMessage) {
     return new ExpectEthAccountsException(transactions.accounts(), expectedMessage);
   }
 
-  public Condition expectSuccessfulTransactionReceipt(final Hash transactionHash) {
+  public Condition expectSuccessfulTransactionReceipt(final String transactionHash) {
     return new ExpectSuccessfulEthGetTransactionReceipt(
-        transactions.getTransactionReceipt(transactionHash.toString()));
+        transactions.getTransactionReceipt(transactionHash));
   }
 
   public Condition expectNoTransactionReceipt(final String transactionHash) {

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/PantheonNode.java

@@ -19,6 +19,7 @@
 import tech.pegasys.pantheon.crypto.SECP256K1.KeyPair;
 import tech.pegasys.pantheon.ethereum.core.Address;
 import tech.pegasys.pantheon.ethereum.core.MiningParameters;
+import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
 import tech.pegasys.pantheon.ethereum.core.Util;
 import tech.pegasys.pantheon.ethereum.jsonrpc.JsonRpcConfiguration;
 import tech.pegasys.pantheon.ethereum.jsonrpc.websocket.WebSocketConfiguration;
@@ -29,6 +30,7 @@
 import tech.pegasys.pantheon.tests.acceptance.dsl.httptransaction.HttpTransaction;
 import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.AdminJsonRpcRequestFactory;
 import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.CliqueJsonRpcRequestFactory;
+import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.EeaJsonRpcRequestFactory;
 import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.IbftJsonRpcRequestFactory;
 import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.JsonRequestFactories;
 import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.PermissioningJsonRpcRequestFactory;
@@ -79,6 +81,7 @@ public class PantheonNode implements Node, NodeConfiguration, RunnableNode, Auto
 
   private final String name;
   private final MiningParameters miningParameters;
+  private final PrivacyParameters privacyParameters;
   private final JsonRpcConfiguration jsonRpcConfiguration;
   private final WebSocketConfiguration webSocketConfiguration;
   private final MetricsConfiguration metricsConfiguration;
@@ -98,6 +101,7 @@ public class PantheonNode implements Node, NodeConfiguration, RunnableNode, Auto
   public PantheonNode(
       final String name,
       final MiningParameters miningParameters,
+      final PrivacyParameters privacyParameters,
       final JsonRpcConfiguration jsonRpcConfiguration,
       final WebSocketConfiguration webSocketConfiguration,
       final MetricsConfiguration metricsConfiguration,
@@ -114,6 +118,7 @@ public PantheonNode(
     this.p2pPort = p2pPort;
     this.name = name;
     this.miningParameters = miningParameters;
+    this.privacyParameters = privacyParameters;
     this.jsonRpcConfiguration = jsonRpcConfiguration;
     this.webSocketConfiguration = webSocketConfiguration;
     this.metricsConfiguration = metricsConfiguration;
@@ -234,6 +239,7 @@ private JsonRequestFactories jsonRequestFactories() {
               new IbftJsonRpcRequestFactory(web3jService),
               new PermissioningJsonRpcRequestFactory(web3jService),
               new AdminJsonRpcRequestFactory(web3jService),
+              new EeaJsonRpcRequestFactory(web3jService),
               websocketService);
     }
 
@@ -292,6 +298,10 @@ public void useAuthenticationTokenInHeaderForJsonRpc(final String token) {
     this.token = token;
   }
 
+  public KeyPair getKeyPair() {
+    return keyPair;
+  }
+
   private void checkIfWebSocketEndpointIsAvailable(final String url) {
     final WebSocketClient webSocketClient = new WebSocketClient(URI.create(url));
     // Web3j implementation always invoke the listener (even when one hasn't been set). We are using
@@ -424,6 +434,10 @@ MiningParameters getMiningParameters() {
     return miningParameters;
   }
 
+  public PrivacyParameters getPrivacyParameters() {
+    return privacyParameters;
+  }
+
   public boolean isDevMode() {
     return devMode;
   }

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/ProcessPantheonNodeRunner.java

@@ -71,6 +71,16 @@ public void startNode(final PantheonNode node) {
       params.add(node.getMiningParameters().getCoinbase().get().toString());
     }
 
+    if (node.getPrivacyParameters().isEnabled()) {
+      params.add("--privacy-enabled");
+      params.add("--privacy-url");
+      params.add(node.getPrivacyParameters().getUrl());
+      params.add("--privacy-public-key-file");
+      params.add(node.getPrivacyParameters().getPublicKeyFile().getAbsolutePath());
+      params.add("--privacy-precompiled-address");
+      params.add(String.valueOf(node.getPrivacyParameters().getPrivacyAddress()));
+    }
+
     params.add("--bootnodes");
     params.add(String.join(",", node.bootnodes().toString()));
 

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/ThreadPantheonNodeRunner.java

@@ -20,7 +20,6 @@
 import tech.pegasys.pantheon.cli.PantheonControllerBuilder;
 import tech.pegasys.pantheon.controller.KeyPairUtil;
 import tech.pegasys.pantheon.controller.PantheonController;
-import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
 import tech.pegasys.pantheon.ethereum.eth.sync.SynchronizerConfiguration;
 import tech.pegasys.pantheon.metrics.MetricsSystem;
 import tech.pegasys.pantheon.metrics.noop.NoOpMetricsSystem;
@@ -69,10 +68,10 @@ public void startNode(final PantheonNode node) {
               .ethNetworkConfig(ethNetworkConfig)
               .syncWithOttoman(false)
               .miningParameters(node.getMiningParameters())
+              .privacyParameters(node.getPrivacyParameters())
               .devMode(node.isDevMode())
               .nodePrivateKeyFile(KeyPairUtil.getDefaultKeyFile(node.homeDirectory()))
               .metricsSystem(noOpMetricsSystem)
-              .privacyParameters(PrivacyParameters.noPrivacy())
               .build();
     } catch (final IOException e) {
       throw new RuntimeException("Error building PantheonController", e);

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/factory/PantheonFactoryConfiguration.java

@@ -13,6 +13,7 @@
 package tech.pegasys.pantheon.tests.acceptance.dsl.node.factory;
 
 import tech.pegasys.pantheon.ethereum.core.MiningParameters;
+import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
 import tech.pegasys.pantheon.ethereum.jsonrpc.JsonRpcConfiguration;
 import tech.pegasys.pantheon.ethereum.jsonrpc.websocket.WebSocketConfiguration;
 import tech.pegasys.pantheon.ethereum.permissioning.PermissioningConfiguration;
@@ -25,6 +26,7 @@ class PantheonFactoryConfiguration {
 
   private final String name;
   private final MiningParameters miningParameters;
+  private final PrivacyParameters privacyParameters;
   private final JsonRpcConfiguration jsonRpcConfiguration;
   private final WebSocketConfiguration webSocketConfiguration;
   private final MetricsConfiguration metricsConfiguration;
@@ -38,6 +40,7 @@ class PantheonFactoryConfiguration {
   PantheonFactoryConfiguration(
       final String name,
       final MiningParameters miningParameters,
+      final PrivacyParameters privacyParameters,
       final JsonRpcConfiguration jsonRpcConfiguration,
       final WebSocketConfiguration webSocketConfiguration,
       final MetricsConfiguration metricsConfiguration,
@@ -49,6 +52,7 @@ class PantheonFactoryConfiguration {
       final boolean isBootnode) {
     this.name = name;
     this.miningParameters = miningParameters;
+    this.privacyParameters = privacyParameters;
     this.jsonRpcConfiguration = jsonRpcConfiguration;
     this.webSocketConfiguration = webSocketConfiguration;
     this.metricsConfiguration = metricsConfiguration;
@@ -68,6 +72,10 @@ public MiningParameters getMiningParameters() {
     return miningParameters;
   }
 
+  public PrivacyParameters getPrivacyParameters() {
+    return privacyParameters;
+  }
+
   public JsonRpcConfiguration getJsonRpcConfiguration() {
     return jsonRpcConfiguration;
   }

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/factory/PantheonFactoryConfigurationBuilder.java

@@ -16,7 +16,9 @@
 
 import tech.pegasys.pantheon.ethereum.core.MiningParameters;
 import tech.pegasys.pantheon.ethereum.core.MiningParametersTestBuilder;
+import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
 import tech.pegasys.pantheon.ethereum.jsonrpc.JsonRpcConfiguration;
+import tech.pegasys.pantheon.ethereum.jsonrpc.RpcApis;
 import tech.pegasys.pantheon.ethereum.jsonrpc.websocket.WebSocketConfiguration;
 import tech.pegasys.pantheon.ethereum.permissioning.PermissioningConfiguration;
 import tech.pegasys.pantheon.metrics.prometheus.MetricsConfiguration;
@@ -32,6 +34,7 @@ public class PantheonFactoryConfigurationBuilder {
   private String name;
   private MiningParameters miningParameters =
       new MiningParametersTestBuilder().enabled(false).build();
+  private PrivacyParameters privacyParameters = PrivacyParameters.noPrivacy();
   private JsonRpcConfiguration jsonRpcConfiguration = JsonRpcConfiguration.createDefault();
   private WebSocketConfiguration webSocketConfiguration = WebSocketConfiguration.createDefault();
   private MetricsConfiguration metricsConfiguration = MetricsConfiguration.createDefault();
@@ -72,6 +75,14 @@ public PantheonFactoryConfigurationBuilder jsonRpcEnabled() {
     return this;
   }
 
+  public PantheonFactoryConfigurationBuilder enablePrivateTransactions(
+      final PrivacyParameters privacyParameters) {
+    this.jsonRpcConfiguration.addRpcApi(RpcApis.EEA);
+    this.privacyParameters = privacyParameters;
+    this.privacyParameters.setEnabled(true);
+    return this;
+  }
+
   public PantheonFactoryConfigurationBuilder jsonRpcAuthenticationEnabled()
       throws URISyntaxException {
     final String authTomlPath =
@@ -156,6 +167,7 @@ public PantheonFactoryConfiguration build() {
     return new PantheonFactoryConfiguration(
         name,
         miningParameters,
+        privacyParameters,
         jsonRpcConfiguration,
         webSocketConfiguration,
         metricsConfiguration,

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/node/factory/PantheonNodeFactory.java

@@ -21,6 +21,7 @@
 import tech.pegasys.pantheon.consensus.clique.CliqueExtraData;
 import tech.pegasys.pantheon.consensus.ibft.IbftExtraData;
 import tech.pegasys.pantheon.ethereum.core.Address;
+import tech.pegasys.pantheon.ethereum.core.PrivacyParameters;
 import tech.pegasys.pantheon.ethereum.jsonrpc.JsonRpcConfiguration;
 import tech.pegasys.pantheon.ethereum.jsonrpc.RpcApi;
 import tech.pegasys.pantheon.ethereum.jsonrpc.RpcApis;
@@ -56,6 +57,7 @@ private PantheonNode create(final PantheonFactoryConfiguration config) throws IO
         new PantheonNode(
             config.getName(),
             config.getMiningParameters(),
+            config.getPrivacyParameters(),
             config.getJsonRpcConfiguration(),
             config.getWebSocketConfiguration(),
             config.getMetricsConfiguration(),
@@ -81,6 +83,18 @@ public PantheonNode createMinerNode(final String name) throws IOException {
             .build());
   }
 
+  public PantheonNode createPrivateTransactionEnabledMinerNode(
+      final String name, final PrivacyParameters privacyParameters) throws IOException {
+    return create(
+        new PantheonFactoryConfigurationBuilder()
+            .setName(name)
+            .miningEnabled()
+            .jsonRpcEnabled()
+            .enablePrivateTransactions(privacyParameters)
+            .webSocketEnabled()
+            .build());
+  }
+
   public PantheonNode createMinerNodeWithCustomRefreshDelay(
       final String name, final Long refreshDelay) throws IOException {
 

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/transaction/EeaJsonRpcRequestFactory.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2019 ConsenSys AG.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+package tech.pegasys.pantheon.tests.acceptance.dsl.transaction;
+
+import tech.pegasys.pantheon.tests.acceptance.dsl.transaction.ResponseTypes.PrivateTransactionReceiptResponse;
+
+import java.util.Collections;
+
+import org.assertj.core.util.Lists;
+import org.web3j.protocol.Web3jService;
+import org.web3j.protocol.core.Request;
+
+public class EeaJsonRpcRequestFactory {
+
+  private final Web3jService web3jService;
+
+  public EeaJsonRpcRequestFactory(final Web3jService web3jService) {
+    this.web3jService = web3jService;
+  }
+
+  public Request<?, org.web3j.protocol.core.methods.response.EthSendTransaction>
+      eeaSendRawTransaction(final String signedTransactionData) {
+    return new Request<>(
+        "eea_sendRawTransaction",
+        Collections.singletonList(signedTransactionData),
+        web3jService,
+        org.web3j.protocol.core.methods.response.EthSendTransaction.class);
+  }
+
+  public Request<?, PrivateTransactionReceiptResponse> eeaGetTransactionReceipt(
+      final String txHash, final String publicKey) {
+    return new Request<>(
+        "eea_getTransactionReceipt",
+        Lists.newArrayList(txHash, publicKey),
+        web3jService,
+        PrivateTransactionReceiptResponse.class);
+  }
+}

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/transaction/JsonRequestFactories.java

@@ -24,6 +24,7 @@ public class JsonRequestFactories {
   private final IbftJsonRpcRequestFactory ibft;
   private final PermissioningJsonRpcRequestFactory perm;
   private final AdminJsonRpcRequestFactory admin;
+  private final EeaJsonRpcRequestFactory eea;
   private final Optional<WebSocketService> websocketService;
 
   public JsonRequestFactories(
@@ -32,12 +33,14 @@ public JsonRequestFactories(
       final IbftJsonRpcRequestFactory ibft,
       final PermissioningJsonRpcRequestFactory perm,
       final AdminJsonRpcRequestFactory admin,
+      final EeaJsonRpcRequestFactory eea,
       final Optional<WebSocketService> websocketService) {
     this.netEth = netEth;
     this.clique = clique;
     this.ibft = ibft;
     this.perm = perm;
     this.admin = admin;
+    this.eea = eea;
     this.websocketService = websocketService;
   }
 
@@ -65,6 +68,10 @@ public AdminJsonRpcRequestFactory admin() {
     return admin;
   }
 
+  public EeaJsonRpcRequestFactory eea() {
+    return eea;
+  }
+
   public void shutdown() {
     netEth.shutdown();
     websocketService.ifPresent(WebSocketService::close);

acceptance-tests/src/test/java/tech/pegasys/pantheon/tests/acceptance/dsl/transaction/ResponseTypes.java

@@ -39,4 +39,39 @@ public static class AddNodeResponse extends Response<String> {}
   public static class RemoveNodeResponse extends Response<String> {}
 
   public static class GetNodesWhitelistResponse extends Response<List<String>> {}
+
+  public static class PrivateTransactionReceiptResponse
+      extends Response<PrivateTransactionReceipt> {}
+
+  public static class PrivateTransactionReceipt {
+    private String contractAddress;
+    private String from;
+    private String to;
+
+    public PrivateTransactionReceipt() {}
+
+    public String getContractAddress() {
+      return contractAddress;
+    }
+
+    public void setContractAddress(final String contractAddress) {
+      this.contractAddress = contractAddress;
+    }
+
+    public String getFrom() {
+      return from;
+    }
+
+    public void setFrom(final String from) {
+      this.from = from;
+    }
+
+    public String getTo() {
+      return to;
+    }
+
+    public void setTo(final String to) {
+      this.to = to;
+    }
+  }
 }