1.0.0

CHANGELOG.md

@@ -1,6 +1,6 @@
-# WIP
+# 1.0.0
 
-- Support `:postgresql`, `:h2`
+- Support `:h2`, `:mysql`, `:postgresql` dbtypes
 
 # 0.1.0
 

README.md

@@ -7,7 +7,7 @@ See [datascript/doc/storage.md](https://github.com/tonsky/datascript/blob/master
 Add this to `deps.edn`:
 
 ```clj
-io.github.tonsky/datascript-storage-sql {:mvn/version "0.1.0"}
+io.github.tonsky/datascript-storage-sql {:mvn/version "1.0.0"}
 ```
 
 Create storage by passing in `java.sql.Connection` and `:dbtype` option:
@@ -65,9 +65,10 @@ or
 
 Currently supported `:dbtype`-s:
 
-- `:sqlite`
 - `:h2`
+- `:mysql`
 - `:postgresql`
+- `:sqlite`
 
 If needed, you can close connection through storage:
 

deps.edn

@@ -3,18 +3,13 @@
   datascript/datascript {:mvn/version "1.5.2"}}
  :aliases
  {:dev
-  {:extra-paths
-   ["dev"]
+  {:extra-paths ["dev"]
    :extra-deps
    {org.clojure/tools.namespace {:mvn/version "1.4.4"}}}
-  :sqlite
-  {:extra-deps
-   {org.xerial/sqlite-jdbc {:mvn/version "3.42.0.0"}}}
-  :h2
-  {:extra-deps
-   {com.h2database/h2 {:mvn/version "2.2.220"}}}
-  :postgresql
-  {:extra-deps
-   {org.postgresql/postgresql {:mvn/version "42.6.0"}}}}}
-
-
+  :test
+  {:extra-paths ["test"]
+   :extra-deps
+   {com.h2database/h2           {:mvn/version "2.2.220"}
+    com.mysql/mysql-connector-j {:mvn/version "8.1.0"}
+    org.postgresql/postgresql   {:mvn/version "42.6.0"}
+    org.xerial/sqlite-jdbc      {:mvn/version "3.42.0.0"}}}}}

dev/data_readers.clj

@@ -0,0 +1 @@
+{p user/p}

project.clj

@@ -1,4 +1,4 @@
-(defproject io.github.tonsky/datascript-storage-sql "0.1.0"
+(defproject io.github.tonsky/datascript-storage-sql "0.0.0"
   :description "SQL Storage implementation for DataScript"
   :license     {:name "MIT" :url "https://github.com/tonsky/datascript-storage-sql/blob/master/LICENSE"}
   :url         "https://github.com/tonsky/datascript-storage-sql"

script/repl.sh

@@ -2,4 +2,4 @@
 set -o errexit -o nounset -o pipefail
 cd "`dirname $0`/.."
 
-clj -A:dev:test:sqlite:h2:postgresql -M -m user
+clj -A:dev:test -M -m user

script/test_all.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+set -o errexit -o nounset -o pipefail
+cd "`dirname $0`/.."
+
+clj -A:test -M -m datascript.storage.sql.test-main

src/datascript/storage/sql/core.clj

@@ -19,36 +19,31 @@
        (finally
          (.setAutoCommit conn# true)))))
 
-(defn execute! [conn sql]
+(defn execute! [^Connection conn sql]
   (with-open [stmt (.createStatement conn)]
     (.execute stmt sql)))
 
-(defmulti store-impl
-  (fn [conn opts addr+data-seq]
-    (:dbtype opts)))
+(defmulti upsert-dml :dbtype)
 
-(defmethod store-impl :h2 [^Connection conn opts addr+data-seq]
-  (let [{:keys [table binary? freeze-str freeze-bytes batch-size]} opts
-        sql (str "merge into " table " key (addr) values (?, ?)")]
-    (with-tx conn
-      (with-open [stmt (.prepareStatement conn sql)]
-        (doseq [part (partition-all batch-size addr+data-seq)]
-          (doseq [[addr data] part]
-            (.setLong stmt 1 addr)
-            (if binary?
-              (let [content ^bytes (freeze-bytes data)]
-                (.setBytes stmt 2 content))
-              (let [content ^String (freeze-str data)]
-                (.setString stmt 2 content)))
-            (.addBatch stmt))
-          (.executeBatch stmt))))))
+(defmethod upsert-dml :h2 [opts]
+  (str "merge into " (:table opts) " key (addr) values (?, ?)"))
+
+(defmethod upsert-dml :mysql [opts]
+  (str
+    "insert into " (:table opts) " (addr, content) "
+    "values (?, ?) "
+    "ON DUPLICATE KEY UPDATE content = ?"))
+
+(defmethod upsert-dml :default [opts]
+  (str
+    "insert into " (:table opts) " (addr, content) "
+    "values (?, ?) "
+    "on conflict(addr) do update set content = ?"))
 
-(defmethod store-impl :default [^Connection conn opts addr+data-seq]
+(defn store-impl [^Connection conn opts addr+data-seq]
   (let [{:keys [table binary? freeze-str freeze-bytes batch-size]} opts
-        sql (str
-              "insert into " table " (addr, content) "
-              "values (?, ?) "
-              "on conflict(addr) do update set content = ?")]
+        sql (upsert-dml opts)
+        cnt (count (re-seq #"\?" sql))]
     (with-tx conn
       (with-open [stmt (.prepareStatement conn sql)]
         (doseq [part (partition-all batch-size addr+data-seq)]
@@ -57,10 +52,12 @@
             (if binary?
               (let [content ^bytes (freeze-bytes data)]
                 (.setBytes stmt 2 content)
-                (.setBytes stmt 3 content))
+                (when (= 3 cnt)
+                  (.setBytes stmt 3 content)))
               (let [content ^String (freeze-str data)]
                 (.setString stmt 2 content)
-                (.setString stmt 3 content)))
+                (when (= 3 cnt)
+                  (.setString stmt 3 content))))
             (.addBatch stmt))
           (.executeBatch stmt))))))
 
@@ -108,6 +105,12 @@
     " (addr BIGINT primary key, "
     "  content " (if binary? "BINARY VARYING" "CHARACTER VARYING") ")"))
 
+(defmethod ddl :mysql [{:keys [table binary?]}]
+  (str
+    "create table if not exists " table
+    " (addr BIGINT primary key, "
+    "  content " (if binary? "LONGBLOB" "LONGTEXT") ")"))
+
 (defmethod ddl :postgresql [{:keys [table binary?]}]
   (str
     "create table if not exists " table

test/datascript/storage/sql/test_core.clj

@@ -3,7 +3,9 @@
     [clojure.edn :as edn]
     [clojure.test :as t :refer [deftest is are testing]]
     [datascript.core :as d]
-    [datascript.storage.sql.core :as storage-sql]))
+    [datascript.storage.sql.core :as storage-sql])
+  (:import
+    [java.sql Connection]))
 
 (defn test-storage [{:keys [dbtype connect-fn reset-fn]}]
   (let [schema {:id {:db/unique :db.unique/identity}}
@@ -13,7 +15,7 @@
                  :age   38}]]
     (reset-fn)
     (testing "Read back same conn"
-      (with-open [conn (connect-fn)]
+      (with-open [conn ^Connection (connect-fn)]
         (let [db      (d/db-with (d/empty-db schema) tx)
               storage (storage-sql/make conn
                         {:dbtype dbtype})
@@ -22,7 +24,7 @@
           (is (= db db')))))
 
     (testing "Read back new conn"
-      (with-open [conn (connect-fn)]
+      (with-open [conn ^Connection (connect-fn)]
         (let [db      (d/db-with (d/empty-db schema) tx)
               storage (storage-sql/make conn
                         {:dbtype dbtype})
@@ -31,7 +33,7 @@
 
     (reset-fn)
     (testing "Rountrip binary"
-      (with-open [conn (connect-fn)]
+      (with-open [conn ^Connection (connect-fn)]
         (let [db      (d/db-with (d/empty-db schema) tx)
               storage (storage-sql/make conn
                         {:dbtype       dbtype
@@ -40,6 +42,3 @@
               _       (d/store db storage)
               db'     (d/restore storage)]
           (is (= db db')))))))
-
-(comment
-  (t/run-all-tests))

test/datascript/storage/sql/test_main.clj

@@ -0,0 +1,14 @@
+(ns datascript.storage.sql.test-main
+  (:require
+    [clojure.test :as t]
+    [datascript.storage.sql.test-core]
+    [datascript.storage.sql.test-h2]
+    [datascript.storage.sql.test-mysql]
+    [datascript.storage.sql.test-postgresql]
+    [datascript.storage.sql.test-sqlite]))
+
+(defn -main [& args]
+  (t/run-all-tests #"datascript\.storage\.sql\..*"))
+
+(comment
+  (-main))

test/datascript/storage/sql/test_mysql.clj

@@ -0,0 +1,24 @@
+(ns datascript.storage.sql.test-mysql
+  (:require
+    [clojure.test :as t :refer [deftest is are testing]]
+    [datascript.core :as d]
+    [datascript.storage.sql.core :as storage-sql]
+    [datascript.storage.sql.test-core :as test-core])
+  (:import
+    [java.nio.file Files Path]
+    [java.sql Connection DriverManager]))
+
+(deftest test-mysql []
+  (let [connect-fn #(DriverManager/getConnection
+                      "jdbc:mysql://localhost:3306/test_datascript?allowPublicKeyRetrieval=true&useSSL=false"
+                      "testuser"
+                      "testpasswd")]
+    (test-core/test-storage
+      {:dbtype     :mysql
+       :reset-fn   #(with-open [conn ^Connection (connect-fn)]
+                      (storage-sql/execute! conn "drop table if exists datascript"))
+       :connect-fn connect-fn})))
+
+(comment
+  (t/run-test-var #'test-mysql)
+  (t/run-tests *ns*))

test/datascript/storage/sql/test_postgresql.clj

@@ -10,7 +10,7 @@
 
 (deftest test-postgresql []
   (test-core/test-storage
-    {:dbtype   :postgresql
+    {:dbtype :postgresql
      :reset-fn
      #(with-open [conn (DriverManager/getConnection "jdbc:postgresql:test_datascript")]
         (storage-sql/execute! conn "drop table if exists datascript"))