Skip to content

Commit

Permalink
datascript.serialize
Browse files Browse the repository at this point in the history
  • Loading branch information
@tonsky
tonsky committed May 31, 2021
1 parent e513e13 commit 4c11a56
Show file tree
Hide file tree
Showing 3 changed files with 222 additions and 7 deletions.
16 changes: 10 additions & 6 deletions src/datascript/db.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,17 @@

(defprotocol IDatom
(datom-tx [this])
(datom-added [this]))
(datom-added [this])
(datom-get-idx [this])
(datom-set-idx [this value]))

(deftype Datom #?(:clj [^int e a v ^int tx ^:unsynchronized-mutable ^int _hash]
:cljs [^number e a v ^number tx ^:mutable ^number _hash])
(deftype Datom #?(:clj [^int e a v ^int tx ^:unsynchronized-mutable ^int idx ^:unsynchronized-mutable ^int _hash]
:cljs [^number e a v ^number tx ^:mutable ^number idx ^:mutable ^number _hash])
IDatom
(datom-tx [d] (if (pos? tx) tx (- tx)))
(datom-added [d] (pos? tx))
(datom-get-idx [_] idx)
(datom-set-idx [_ value] (set! idx (int value)))

#?@(:cljs
[IHash
Expand Down Expand Up @@ -224,9 +228,9 @@
#?(:cljs (goog/exportSymbol "datascript.db.Datom" Datom))

(defn ^Datom datom
([e a v] (Datom. e a v tx0 0))
([e a v tx] (Datom. e a v tx 0))
([e a v tx added] (Datom. e a v (if added tx (- tx)) 0)))
([e a v] (Datom. e a v tx0 0 0))
([e a v tx] (Datom. e a v tx 0 0))
([e a v tx added] (Datom. e a v (if added tx (- tx)) 0 0)))

(defn datom? [x] (instance? Datom x))

Expand Down
2 changes: 1 addition & 1 deletion src/datascript/externs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ datascript.db.Datom.prototype.e;
datascript.db.Datom.prototype.a;
datascript.db.Datom.prototype.v;
datascript.db.Datom.prototype.tx;

datascript.db.Datom.prototype.idx;

datascript.impl = {};
datascript.impl.entity = {};
Expand Down
211 changes: 211 additions & 0 deletions src/datascript/serialize.cljc
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
(ns datascript.serialize
(:refer-clojure :exclude [amap])
(:require
[clojure.edn :as edn]
[clojure.string :as str]
[datascript.db :as db #?(:cljs :refer-macros :clj :refer) [raise cond+] #?@(:cljs [:refer [Datom]])]
[me.tonsky.persistent-sorted-set :as set]
[me.tonsky.persistent-sorted-set.arrays :as arrays])
#?(:cljs (:require-macros [datascript.serialize :refer [array dict]]))
#?(:clj
(:import
[datascript.db Datom])))

(def ^:const marker-kw 0)
(def ^:const marker-other 1)

(defn- if-cljs [env then else]
(if (:ns env) then else))

#?(:clj
(defmacro array
"Platform-native array representation (java.util.List on JVM, Array on JS)"
[& args]
(if-cljs &env
(list* 'js* (str "[" (str/join "," (repeat (count args) "~{}")) "]") args)
`(java.util.List/of ~@args))))

#?(:clj
(defmacro dict
"Platform-native dictionary representation (java.util.Map on JVM, Object on JS)"
[& args]
(if-cljs &env
(list* 'js* (str "{" (str/join "," (repeat (/ (count args) 2) "~{}:~{}")) "}") args)
`(array-map ~@args))))

(defn- array-get [d i]
#?(:clj (.get ^java.util.List d (int i))
:cljs (arrays/aget d i)))

(defn- dict-get [d k]
#?(:clj (.get ^java.util.Map d k)
:cljs (arrays/aget d k)))

(defn- amap [f xs]
#?(:clj
(let [arr (java.util.ArrayList. (count xs))]
(reduce (fn [idx x] (.add arr (f x)) (inc idx)) 0 xs)
arr)
:cljs
(let [arr (js/Array. (count xs))]
(reduce (fn [idx x] (arrays/aset arr idx (f x)) (inc idx)) 0 xs)
arr)))

(defn- amap-indexed [f xs]
#?(:clj
(let [arr (java.util.ArrayList. (count xs))]
(reduce (fn [idx x] (.add arr (f idx x)) (inc idx)) 0 xs)
arr)
:cljs
(let [arr (js/Array. (count xs))]
(reduce (fn [idx x] (arrays/aset arr idx (f idx x)) (inc idx)) 0 xs)
arr)))

(defn- attr-comparator
"Looks for a datom with attribute exactly bigger than the given one"
[^Datom d1 ^Datom d2]
(cond
(nil? (.-a d2)) -1
(<= (compare (.-a d1) (.-a d2)) 0) -1
true 1))

(defn- all-attrs
"All attrs in a DB, distinct, sorted"
[db]
(if (empty? (:aevt db))
[]
(loop [attrs (transient [(:a (first (:aevt db)))])]
(let [attr (nth attrs (dec (count attrs)))
left (db/datom 0 attr nil)
right (db/datom db/emax nil nil)
next-attr (:a (first (set/slice (:aevt db) left right attr-comparator)))]
(if (some? next-attr)
(recur (conj! attrs next-attr))
(persistent! attrs))))))

(def ^{:arglists '([kw])} freeze-kw str)

(defn thaw-kw [s]
(if (str/starts-with? s ":")
(keyword (subs s 1))
s))

(defn ^:export serializable
"Converts db into a data structure (not string!) that can be fed to JSON
serializer of your choice (`js/JSON.stringify` in CLJS, `cheshire.core/generate-string` or
`jsonista.core/write-value-as-string` in CLJ).
Options:
Non-primitive values will be serialized using optional :freeze-fn (`pr-str` by default).
Serialized structure breakdown:
count :: number
tx0 :: number
max-eid :: number
max-tx :: number
schema :: freezed :schema
attrs :: [keywords ...]
keywords :: [keywords ...]
eavt :: [[e a-idx v dtx] ...]
a-idx :: index in attrs
v :: (string | number | boolean | [0 <index in keywords>] | [1 <freezed v>])
dtx :: tx - tx0
aevt :: [<index in eavt> ...]
avet :: [<index in eavt> ...]"
([db]
(serializable db {}))
([db {:keys [freeze-fn]
:or {freeze-fn pr-str}}]
(let [attrs (all-attrs db)
attrs-map (into {} (map vector attrs (range)))
*kws (volatile! (transient []))
*kw-map (volatile! (transient {}))
write-kw (fn [kw]
(let [idx (or
(get @*kw-map kw)
(let [keywords (vswap! *kws conj! kw)
idx (dec (count keywords))]
(vswap! *kw-map assoc! kw idx)
idx))]
(array marker-kw idx)))
write-other (fn [v] (array marker-other (freeze-fn v)))
write-v (fn [v]
(cond
(string? v) v
#?@(:clj [(ratio? v) (write-other v)])
(number? v) v
(boolean? v) v
(keyword? v) (write-kw v)
:else (write-other v)))
eavt (amap-indexed
(fn [idx ^Datom d]
(db/datom-set-idx d idx)
(let [e (.-e d)
a (attrs-map (.-a d))
v (write-v (.-v d))
tx (- (.-tx d) db/tx0)]
(array e a v tx)))
(:eavt db))
aevt (amap-indexed (fn [_ ^Datom d] (db/datom-get-idx d)) (:aevt db))
avet (amap-indexed (fn [_ ^Datom d] (db/datom-get-idx d)) (:avet db))
schema (freeze-fn (:schema db))
attrs (amap freeze-kw attrs)
kws (amap freeze-kw (persistent! @*kws))]
(dict
"count" (count (:eavt db))
"tx0" db/tx0
"max-eid" (:max-eid db)
"max-tx" (:max-tx db)
"schema" schema
"attrs" attrs
"keywords" kws
"eavt" eavt
"aevt" aevt
"avet" avet))))

(defn ^:export from-serializable
"Creates db from a data structure (not string!) produced by serializable.
Non-primitive values will be deserialized using optional :thaw-fn
(`clojure.edn/read-string` by default).
:thaw-fn must match :freeze-fn from serializable."
([serializable]
(from-serializable serializable {}))
([serializable {:keys [thaw-fn]
:or {thaw-fn edn/read-string}}]
(let [tx0 (dict-get serializable "tx0")
schema (thaw-fn (dict-get serializable "schema"))
_ (#'db/validate-schema schema)
attrs (->> (dict-get serializable "attrs") (mapv thaw-kw))
keywords (->> (dict-get serializable "keywords") (mapv thaw-kw))
eavt (->> (dict-get serializable "eavt")
(amap (fn [arr]
(let [e (array-get arr 0)
a (nth attrs (array-get arr 1))
v (array-get arr 2)
v (cond
(number? v) v
(string? v) v
(boolean? v) v
(arrays/array? v)
(let [marker (array-get v 0)]
(case marker
marker-kw (array-get keywords (array-get v 1))
marker-other (thaw-fn (array-get v 1)))))
tx (+ tx0 (array-get arr 3))]
(db/datom e a v tx))))
#?(:clj .toArray))
aevt (some->> (dict-get serializable "aevt") (amap #(arrays/aget eavt %)) #?(:clj .toArray))
avet (some->> (dict-get serializable "avet") (amap #(arrays/aget eavt %)) #?(:clj .toArray))]
(db/map->DB
{:schema schema
:rschema (#'db/rschema (merge db/implicit-schema schema))
:eavt (set/from-sorted-array db/cmp-datoms-eavt eavt)
:aevt (set/from-sorted-array db/cmp-datoms-aevt aevt)
:avet (set/from-sorted-array db/cmp-datoms-avet avet)
:max-eid (dict-get serializable "max-eid")
:max-tx (dict-get serializable "max-tx")
:hash (atom 0)}))))

0 comments on commit 4c11a56

Please sign in to comment.