Add relational database store (Consensys#194)

* Move to latest cava build

* Add a SQL database store

build.gradle

@@ -223,6 +223,7 @@ dependencies {
   // storage
   compile 'org.fusesource.leveldbjni:leveldbjni-all:1.8'
   compile 'org.mapdb:mapdb:3.0.7'
+  compile 'com.jolbox:bonecp:0.8.0.RELEASE'
 
   // serialization
   compile 'com.moandjiezana.toml:toml4j:0.7.2'
@@ -236,6 +237,7 @@ dependencies {
   compile 'org.apache.logging.log4j:log4j-1.2-api:2.11.0'
 
   // testing
+  testCompile 'com.h2database:h2:1.4.197'
   testCompile 'org.junit.jupiter:junit-jupiter-api:5.2.0'
   testCompile 'org.junit.jupiter:junit-jupiter-params:5.2.0'
   testRuntime 'org.junit.jupiter:junit-jupiter-engine:5.2.0'

documentation/install/configure.md

@@ -40,4 +40,41 @@ privatekeys = ["foo.key"]
 ```
 
 You can check all the available properties in the  
-[`sample.conf`](https://github.com/ConsenSys/orion/blob/master/src/main/resources/sample.conf) file.
+[`sample.conf`](https://github.com/ConsenSys/orion/blob/master/src/main/resources/sample.conf) file.
+
+### Storage
+
+By default, Orion relies on leveldb to store information.
+
+#### LevelDB
+
+```
+storage = "leveldb:oriondb"
+```
+
+#### MapDB
+
+Orion offers persistence using [MapDB](http://www.mapdb.org/).
+
+```
+storage = "mapdb:oriondb"
+```
+
+#### SQL
+
+Orion supports working with relational databases.
+
+* Add the SQL driver jar to the lib folder of the Orion installation
+* Create a table in your database:
+
+| Database | Create statement |
+|---|---|
+| MySQL | `create table store(key varbinary, value varbinary, primary key(key))` |
+| PostgresQL | `create table store(key bytea, value bytea, primary key(key))` |
+
+* Set storage to:
+
+```
+storage = "sql:jdbc:postgresql://localhost/oriondb"
+
+```

gradle/check-licenses.gradle

@@ -86,6 +86,7 @@ downloadLicenses {
       'Apache License Version 2.0',
       'Apache License, Version 2.0',
       'Apache Software Licenses',
+      'Apache v2',
       'ASL, Version 2',
       'The Apache License, Version 2.0',
       'The Apache Software License, Version 2.0',

src/acceptance-test/java/net/consensys/orion/acceptance/NodeUtils.java

@@ -61,7 +61,8 @@ public static Config nodeConfig(
       String privKeys,
       String tls,
       String tlsServerTrust,
-      String tlsClientTrust) throws IOException {
+      String tlsClientTrust,
+      String storage) throws IOException {
 
     Path workDir = tempDir.resolve("acceptance").toAbsolutePath();
     Files.createDirectories(workDir);
@@ -77,7 +78,7 @@ public static Config nodeConfig(
         + "clienturl = \"" + clientUrl + "\"\n"
         + "clientport = " + clientPort + "\n"
         + "clientnetworkinterface = \"" + clientNetworkInterface + "\"\n"
-        + "storage = \"leveldb:database/" + nodeName + "\"\n"
+        + "storage = \"" + storage + "\"\n"
         + "othernodes = [\"" + otherNodes + "\"]\n"
         + "publickeys = [" + pubKeys + "]\n"
         + "privatekeys = [" + privKeys + "]\n"

src/acceptance-test/java/net/consensys/orion/acceptance/send/SingleNodeSendTest.java

@@ -82,7 +82,8 @@ static void setUpSingleNode(@TempDirectory Path tempDir) throws Exception {
         joinPathsAsTomlListEntry(key1key, key2key),
         "off",
         "tofu",
-        "tofu");
+        "tofu",
+        "leveldb:database/node1");
   }
 
   @BeforeEach

src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/DualNodesSendReceiveTest.java

@@ -37,6 +37,9 @@
 
 import java.net.URL;
 import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
 import java.util.concurrent.TimeUnit;
 
 import io.vertx.core.Vertx;
@@ -84,6 +87,12 @@ void setUpDualNodes(@TempDirectory Path tempDir) throws Exception {
     Path key2pub = copyResource("key2.pub", tempDir.resolve("key2.pub"));
     Path key2key = copyResource("key2.key", tempDir.resolve("key2.key"));
 
+    String jdbcUrl = "jdbc:h2:" + tempDir.resolve("node2").toString();
+    try (Connection conn = DriverManager.getConnection(jdbcUrl)) {
+      Statement st = conn.createStatement();
+      st.executeUpdate("create table if not exists store(key binary, value binary, primary key(key))");
+    }
+
     firstNodeConfig = NodeUtils.nodeConfig(
         tempDir,
         firstNodeBaseUrl,
@@ -98,7 +107,8 @@ void setUpDualNodes(@TempDirectory Path tempDir) throws Exception {
         joinPathsAsTomlListEntry(key1key),
         "off",
         "tofu",
-        "tofu");
+        "tofu",
+        "leveldb:database/node1");
     secondNodeConfig = NodeUtils.nodeConfig(
         tempDir,
         secondNodeBaseUrl,
@@ -113,7 +123,8 @@ void setUpDualNodes(@TempDirectory Path tempDir) throws Exception {
         joinPathsAsTomlListEntry(key2key),
         "off",
         "tofu",
-        "tofu");
+        "tofu",
+        "sql:" + jdbcUrl);
     vertx = vertx();
     firstOrionLauncher = NodeUtils.startOrion(firstNodeConfig);
     firstHttpClient = vertx.createHttpClient();

src/acceptance-test/java/net/consensys/orion/acceptance/send/receive/SingleNodeSendReceiveTest.java

@@ -80,7 +80,8 @@ static void setUpSingleNode(@TempDirectory Path tempDir) throws Exception {
         joinPathsAsTomlListEntry(key1key, key2key),
         "off",
         "tofu",
-        "tofu");
+        "tofu",
+        "leveldb:database/node1");
   }
 
   @BeforeEach

src/main/java/net/consensys/orion/cmd/Orion.java

@@ -23,6 +23,7 @@
 import net.consensys.cava.kv.KeyValueStore;
 import net.consensys.cava.kv.LevelDBKeyValueStore;
 import net.consensys.cava.kv.MapDBKeyValueStore;
+import net.consensys.cava.kv.SQLKeyValueStore;
 import net.consensys.cava.net.tls.TLS;
 import net.consensys.cava.net.tls.VertxTrustOptions;
 import net.consensys.orion.config.Config;
@@ -438,18 +439,23 @@ private KeyValueStore createStorage(String storage, Path storagePath) {
       db = storageOptions[1];
     }
 
-    Path dbPath = storagePath.resolve(db);
     if (storage.toLowerCase().startsWith("mapdb")) {
       try {
-        return MapDBKeyValueStore.open(dbPath);
+        return MapDBKeyValueStore.open(storagePath.resolve(db));
       } catch (IOException e) {
-        throw new OrionStartException("Couldn't create MapDB store: " + dbPath, e);
+        throw new OrionStartException("Couldn't create MapDB store: " + db, e);
       }
     } else if (storage.toLowerCase().startsWith("leveldb")) {
       try {
-        return LevelDBKeyValueStore.open(dbPath);
+        return LevelDBKeyValueStore.open(storagePath.resolve(db));
       } catch (IOException e) {
-        throw new OrionStartException("Couldn't create LevelDB store: " + dbPath, e);
+        throw new OrionStartException("Couldn't create LevelDB store: " + db, e);
+      }
+    } else if (storage.toLowerCase().startsWith("sql")) {
+      try {
+        return SQLKeyValueStore.open(db);
+      } catch (IOException e) {
+        throw new OrionStartException("Couldn't create SQL-backed store: " + db, e);
       }
     } else {
       throw new OrionStartException("unsupported storage mechanism: " + storage);

src/main/java/net/consensys/orion/config/Config.java

@@ -243,6 +243,7 @@ public Optional<Path> passwords() {
    * <ul>
    * <li>leveldb:path - LevelDB
    * <li>mapdb:path - MapDB
+   * <li>sql:jdbcurl - Relational database</li>
    * <li>memory - Contents are cleared when Orion exits
    * </ul>
    *
@@ -523,6 +524,7 @@ private static Schema configSchema() {
             + "\n"
             + "   - leveldb:path - LevelDB\n"
             + "   - mapdb:path - MapDB\n"
+            + "   - sql:jdbcurl - SQL database\n"
             + "   - memory - Contents are cleared when Orion exits",
         Config::validateStorage);
 
@@ -663,10 +665,10 @@ private static List<ConfigurationError> validateStorage(
       @Nullable String value) {
     assert (value != null);
     String storageType = value.split(":", 2)[0];
-    if (!Arrays.asList("mapdb", "leveldb", "memory").contains(storageType)) {
+    if (!Arrays.asList("mapdb", "leveldb", "sql", "memory").contains(storageType)) {
       return singleError(
           position,
-          "Value of property '" + key + "' must have storage type of \"leveldb\", \"mapdb\", or \"memory\"");
+          "Value of property '" + key + "' must have storage type of \"leveldb\", \"mapdb\", \"sql\" or \"memory\"");
     }
     return noErrors();
   }

src/test/java/net/consensys/orion/config/TomlConfigTest.java

@@ -92,7 +92,7 @@ void invalidConfigsThrowException() {
         ConfigException.class,
         () -> Config.load(this.getClass().getClassLoader().getResourceAsStream("invalidConfigTest.toml")));
     String message = "Value of property 'clienturl' is not a valid URL (line 4, column 1)\n"
-        + "Value of property 'storage' must have storage type of \"leveldb\", \"mapdb\", or \"memory\" (line 11, column 1)\n"
+        + "Value of property 'storage' must have storage type of \"leveldb\", \"mapdb\", \"sql\" or \"memory\" (line 11, column 1)\n"
         + "Value of property 'othernodes' is not a valid URL (line 6, column 1)\n"
         + "Value of property 'othernodes' is not a valid URL (line 6, column 1)\n"
         + "Value of property 'tlsservertrust' should be \"whitelist\", \"ca\", \"ca-or-whitelist\", \"tofu\", \"insecure-tofa\", \"ca-or-tofu\", \"insecure-ca-or-tofa\", \"insecure-no-validation\", \"insecure-record\", or \"insecure-ca-or-record\" (line 9, column 1)\n"
@@ -126,6 +126,7 @@ void storageValidationTypes() {
     Config.load("storage=\"memory\"");
     Config.load("storage=\"leveldb\"");
     Config.load("storage=\"mapdb\"");
+    Config.load("storage=\"sql:url\"");
     assertThrows(ConfigException.class, () -> Config.load("storage=\"memoryX\""));
     assertThrows(ConfigException.class, () -> Config.load("storage=\"invalidStorage\""));
   }