From 304ea0c8d01d38e882714664980c77cbedda527f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 26 Jan 2014 11:10:55 +0200 Subject: [PATCH 001/102] adding as an eclipse proj - need to ignore the env --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index a3eebe14c..f4bc9edd7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *~ _site +.project From 4895db6edf288148bece76a4664df6ff1881f006 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 26 Jan 2014 11:40:41 +0200 Subject: [PATCH 002/102] ignoring fetched modules --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f4bc9edd7..063bdbcba 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ _site .project +functionalDB/node_modules From 4ed35d6776eff9b298a81c4e6b986acac02b6875 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 27 Jan 2014 17:24:46 +0200 Subject: [PATCH 003/102] initial structure of the db project --- functionalDB/app.js | 34 ++++++++++++++++++++++ functionalDB/jsomic.js | 5 ++++ functionalDB/package.json | 22 ++++++++++++++ functionalDB/public/stylesheets/.gitignore | 1 + functionalDB/routes/.gitignore | 2 ++ functionalDB/views/.gitignore | 2 ++ 6 files changed, 66 insertions(+) create mode 100644 functionalDB/app.js create mode 100644 functionalDB/jsomic.js create mode 100644 functionalDB/package.json create mode 100644 functionalDB/public/stylesheets/.gitignore create mode 100644 functionalDB/routes/.gitignore create mode 100644 functionalDB/views/.gitignore diff --git a/functionalDB/app.js b/functionalDB/app.js new file mode 100644 index 000000000..d0ac93eba --- /dev/null +++ b/functionalDB/app.js @@ -0,0 +1,34 @@ + +/** + * Module dependencies. + */ + +var express = require('express'); +var routes = require('./routes'); +var http = require('http'); +var path = require('path'); + + +var app = express(); + +// all environments +app.set('port', process.env.PORT || 3000); +app.set('views', __dirname + '/views'); +//app.set('view engine', 'jade'); +app.use(express.favicon()); +app.use(express.logger('dev')); +app.use(express.bodyParser()); +app.use(express.methodOverride()); +app.use(app.router); +app.use(express.static(path.join(__dirname, 'public'))); + +// development only +if ('development' == app.get('env')) { + app.use(express.errorHandler()); +} + +app.get('/', routes.index); + +http.createServer(app).listen(app.get('port'), function(){ + console.log('Express server listening on port ' + app.get('port')); +}); diff --git a/functionalDB/jsomic.js b/functionalDB/jsomic.js new file mode 100644 index 000000000..082051948 --- /dev/null +++ b/functionalDB/jsomic.js @@ -0,0 +1,5 @@ +/** + * New node file + */ + +var mori = require("mori"); \ No newline at end of file diff --git a/functionalDB/package.json b/functionalDB/package.json new file mode 100644 index 000000000..aa790b6dc --- /dev/null +++ b/functionalDB/package.json @@ -0,0 +1,22 @@ +{ + "name": "Jsomic", + "version": "0.1.0", + "scripts": { + "test": "echo \"Error: no test specified! Configure in package.json\" && exit 1", + "start": "node app.js" + }, + "dependencies": { + "mori":"*", + "express": "3.2.6" + }, + + "repository": "", + "keywords": [ + "Node.js", + "Eclipse", + "Nodeclipse" + ], + "author": "", + "license": "MIT", + "readmeFilename": "README.md" +} diff --git a/functionalDB/public/stylesheets/.gitignore b/functionalDB/public/stylesheets/.gitignore new file mode 100644 index 000000000..911321dc0 --- /dev/null +++ b/functionalDB/public/stylesheets/.gitignore @@ -0,0 +1 @@ +/style.css diff --git a/functionalDB/routes/.gitignore b/functionalDB/routes/.gitignore new file mode 100644 index 000000000..46f1114d1 --- /dev/null +++ b/functionalDB/routes/.gitignore @@ -0,0 +1,2 @@ +/index.js +/user.js diff --git a/functionalDB/views/.gitignore b/functionalDB/views/.gitignore new file mode 100644 index 000000000..c654461e8 --- /dev/null +++ b/functionalDB/views/.gitignore @@ -0,0 +1,2 @@ +/index.jade +/layout.jade From eaff46d40fdd2bc5185554f6beba239e98655672 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 8 Feb 2014 22:15:13 +0200 Subject: [PATCH 004/102] changing from node app to clojure --- functionalDB/app.js | 34 ---------------------------------- functionalDB/jsomic.js | 5 ----- functionalDB/package.json | 22 ---------------------- 3 files changed, 61 deletions(-) delete mode 100644 functionalDB/app.js delete mode 100644 functionalDB/jsomic.js delete mode 100644 functionalDB/package.json diff --git a/functionalDB/app.js b/functionalDB/app.js deleted file mode 100644 index d0ac93eba..000000000 --- a/functionalDB/app.js +++ /dev/null @@ -1,34 +0,0 @@ - -/** - * Module dependencies. - */ - -var express = require('express'); -var routes = require('./routes'); -var http = require('http'); -var path = require('path'); - - -var app = express(); - -// all environments -app.set('port', process.env.PORT || 3000); -app.set('views', __dirname + '/views'); -//app.set('view engine', 'jade'); -app.use(express.favicon()); -app.use(express.logger('dev')); -app.use(express.bodyParser()); -app.use(express.methodOverride()); -app.use(app.router); -app.use(express.static(path.join(__dirname, 'public'))); - -// development only -if ('development' == app.get('env')) { - app.use(express.errorHandler()); -} - -app.get('/', routes.index); - -http.createServer(app).listen(app.get('port'), function(){ - console.log('Express server listening on port ' + app.get('port')); -}); diff --git a/functionalDB/jsomic.js b/functionalDB/jsomic.js deleted file mode 100644 index 082051948..000000000 --- a/functionalDB/jsomic.js +++ /dev/null @@ -1,5 +0,0 @@ -/** - * New node file - */ - -var mori = require("mori"); \ No newline at end of file diff --git a/functionalDB/package.json b/functionalDB/package.json deleted file mode 100644 index aa790b6dc..000000000 --- a/functionalDB/package.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "name": "Jsomic", - "version": "0.1.0", - "scripts": { - "test": "echo \"Error: no test specified! Configure in package.json\" && exit 1", - "start": "node app.js" - }, - "dependencies": { - "mori":"*", - "express": "3.2.6" - }, - - "repository": "", - "keywords": [ - "Node.js", - "Eclipse", - "Nodeclipse" - ], - "author": "", - "license": "MIT", - "readmeFilename": "README.md" -} From 8025321db90f12913ff14e152f0e82d7633dbf47 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 8 Feb 2014 22:15:58 +0200 Subject: [PATCH 005/102] changing from node app tp clojure --- .classpath | 6 + .gitignore | 1 + functionalDB/.gitignore | 1 + functionalDB/README.md | 5 + functionalDB/public/stylesheets/.gitignore | 1 - functionalDB/routes/.gitignore | 2 - functionalDB/src/core/fdb.clj | 123 +++++++++++++++++++++ functionalDB/views/.gitignore | 2 - 8 files changed, 136 insertions(+), 5 deletions(-) create mode 100644 .classpath create mode 100644 functionalDB/.gitignore create mode 100644 functionalDB/README.md delete mode 100644 functionalDB/public/stylesheets/.gitignore delete mode 100644 functionalDB/routes/.gitignore create mode 100644 functionalDB/src/core/fdb.clj delete mode 100644 functionalDB/views/.gitignore diff --git a/.classpath b/.classpath new file mode 100644 index 000000000..96d86407f --- /dev/null +++ b/.classpath @@ -0,0 +1,6 @@ + + + + + + diff --git a/.gitignore b/.gitignore index f5b50c496..f5ceb8409 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ functionalDB/node_modules *.pyc +/bin diff --git a/functionalDB/.gitignore b/functionalDB/.gitignore new file mode 100644 index 000000000..5e56e040e --- /dev/null +++ b/functionalDB/.gitignore @@ -0,0 +1 @@ +/bin diff --git a/functionalDB/README.md b/functionalDB/README.md new file mode 100644 index 000000000..c93db369d --- /dev/null +++ b/functionalDB/README.md @@ -0,0 +1,5 @@ +# Jsomic + +Jsomic is an in-memory, no-sql functional database, written in Javascript. + +It is a modest attempt to provide part of the functionality that the Datomic database provides (the main omitted functionality is the durability part). \ No newline at end of file diff --git a/functionalDB/public/stylesheets/.gitignore b/functionalDB/public/stylesheets/.gitignore deleted file mode 100644 index 911321dc0..000000000 --- a/functionalDB/public/stylesheets/.gitignore +++ /dev/null @@ -1 +0,0 @@ -/style.css diff --git a/functionalDB/routes/.gitignore b/functionalDB/routes/.gitignore deleted file mode 100644 index 46f1114d1..000000000 --- a/functionalDB/routes/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/index.js -/user.js diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj new file mode 100644 index 000000000..afe28573f --- /dev/null +++ b/functionalDB/src/core/fdb.clj @@ -0,0 +1,123 @@ +(ns core.fdb) + +(defrecord Entity [e_id name attrs]) +(defrecord Attr [name type value ts prev-ts]) + + +(defn make-entity [name] (Entity. (keyword name) name {})) +(defn make-attr[name val type] (Attr. name type val -1 -1)) +(defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) + +;(defn update-creation-ts [ent ts] +; (let ks [k]) +; +; +; ) + +(defn add-entity[db ent] (let [ + ts-mp (last (:timestamped db)) + new-eavt (assoc (:EAVT ts-mp) (:e_id ent) ent) + new-avet (:AVET ts-mp) + new-ts (assoc ts-mp :AVET new-avet :EAVT new-eavt )] + (assoc db :timestamped (conj (:timestamped db) new-ts)) + )) + +(defn make-db[] + (atom {:timestamped [{:EAVT {} :AVET {}}] + :topId 0 + :curr-time 0 + } + ) + ) + + + +(def db1 (make-db)) + + +(def en (make-entity "hotel")) + +(add-attr en (make-attr "room" 12 :number)) + +(swap! db1 add-entity en) + + + + + + + +(defn recent-ts-val [db](last (:timestamped db))) + + +;(defn update-ts-with-EAV [ts &[e a v] :as more] +; (let[eavt (:EAVT ts)] +; (assoc-in ts :EAVT e a v) +; ) +; ) + +;(defn _add-timestamp[db & more] +; (let [ ts {:EAVT (:EAVT (recent-ts-val db))}] +; (update-ts-with-EAV ts more) +; +; (assoc db :timestamped (conj (:timestamped db) ts)) +; ) +; ) + + + +@db1 + +;(swap! db1 _add-timestamp) + +;@db1 + + + + +(defn next-id [db] (let [curr (@db :curr-time)] + (swap! db assoc :curr-time (inc curr)) + curr)) + + +(defn make-entity[db name] (Entity. (db :curr-time) name)) + +(defn add-entity [db ent] + (let [ + timestamped-vec (:timestamped db) + eavt-map (:EAVT (recent-ts-val db)) + ] + (assoc db :timestamped (conj timestamped-vec {:EAVT + (assoc eavt-map (keyword (:name ent)) ent)} ) ) + ) + ) + + + @db1 + (add-entity @db1 + (make-entity @db1 "person") + ) + ;(add-attr @db1 "name" :string "Jim" ) + ; (add-attr @db1 "sur-name" :string "Doe" ) + ; (add-attr @db1 "age-name" :number 39 ) + + + + + + + + +(next-id db1) +(next-id db1) +;db +(defn fact[n] + (if (<= n 0) 1 + (* n (fact (dec n))) + ) + + ) + +(fact 1) + +;(swap! db assoc :6 5 diff --git a/functionalDB/views/.gitignore b/functionalDB/views/.gitignore deleted file mode 100644 index c654461e8..000000000 --- a/functionalDB/views/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/index.jade -/layout.jade From 251f0ee0e78288549303b1a9adac82e84b3075a6 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 9 Feb 2014 12:29:13 +0200 Subject: [PATCH 006/102] cleanup and timestamp commiting --- functionalDB/src/core/fdb.clj | 96 +++++++++++++++-------------------- 1 file changed, 40 insertions(+), 56 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index afe28573f..e2dbe20fc 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -7,19 +7,31 @@ (defn make-entity [name] (Entity. (keyword name) name {})) (defn make-attr[name val type] (Attr. name type val -1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) +(defn next-ts [db] (inc (:curr-time db))) -;(defn update-creation-ts [ent ts] -; (let ks [k]) -; -; -; ) +(defn update-creation-ts [ent tsVal] + (let [ks (keys (:attrs ent)) + vls (vals (:attrs ent)) + updatedAttrsVals (map #(assoc % :ts tsVal) vls) + updatedAttrs (interleave ks updatedAttrsVals) + ] + (assoc ent :attrs updatedAttrs) + + ) + + ) + +;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [ + new-ts (next-ts db) ts-mp (last (:timestamped db)) - new-eavt (assoc (:EAVT ts-mp) (:e_id ent) ent) + new-eavt (assoc (:EAVT ts-mp) (:e_id ent) (update-creation-ts ent new-ts) ) new-avet (:AVET ts-mp) - new-ts (assoc ts-mp :AVET new-avet :EAVT new-eavt )] - (assoc db :timestamped (conj (:timestamped db) new-ts)) + new-timesampedVal (assoc ts-mp :AVET new-avet :EAVT new-eavt )] + (assoc db + :timestamped (conj (:timestamped db) new-timesampedVal) + :curr-time new-ts ) )) (defn make-db[] @@ -34,20 +46,30 @@ (def db1 (make-db)) +(def en1 (-> (make-entity "hotel") -(def en (make-entity "hotel")) + (add-attr (make-attr "room" 12 :number)) + (add-attr (make-attr "address" "where" :string))) -(add-attr en (make-attr "room" 12 :number)) + ) + +(def en2 (-> (make-entity "book") + + (add-attr (make-attr "length" -1 :number)) + (add-attr (make-attr "auother" "jon" :string))) + + ) -(swap! db1 add-entity en) +(swap! db1 add-entity en1) +(swap! db1 add-entity en2) -(defn recent-ts-val [db](last (:timestamped db))) +;(defn recent-ts-val [db](last (:timestamped db))) ;(defn update-ts-with-EAV [ts &[e a v] :as more] @@ -66,58 +88,20 @@ -@db1 - -;(swap! db1 _add-timestamp) - -;@db1 - - - -(defn next-id [db] (let [curr (@db :curr-time)] - (swap! db assoc :curr-time (inc curr)) - curr)) - - -(defn make-entity[db name] (Entity. (db :curr-time) name)) - -(defn add-entity [db ent] - (let [ - timestamped-vec (:timestamped db) - eavt-map (:EAVT (recent-ts-val db)) - ] - (assoc db :timestamped (conj timestamped-vec {:EAVT - (assoc eavt-map (keyword (:name ent)) ent)} ) ) - ) - ) - - - @db1 - (add-entity @db1 - (make-entity @db1 "person") - ) ;(add-attr @db1 "name" :string "Jim" ) ; (add-attr @db1 "sur-name" :string "Doe" ) ; (add-attr @db1 "age-name" :number 39 ) - - - - - - -(next-id db1) -(next-id db1) ;db -(defn fact[n] - (if (<= n 0) 1 - (* n (fact (dec n))) - ) +;; (defn fact[n] +;; (if (<= n 0) 1 +;; (* n (fact (dec n))) +;; ) - ) +;; ) -(fact 1) +;; (fact 1) ;(swap! db assoc :6 5 From 056ec2b57f972169cfc366ca8466788a2fd998d1 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 9 Feb 2014 12:48:43 +0200 Subject: [PATCH 007/102] better ids of entities --- functionalDB/src/core/fdb.clj | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index e2dbe20fc..a127ad2f4 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -4,7 +4,7 @@ (defrecord Attr [name type value ts prev-ts]) -(defn make-entity [name] (Entity. (keyword name) name {})) +(defn make-entity [name] (Entity. :no-id-yet name {})) (defn make-attr[name val type] (Attr. name type val -1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) @@ -24,14 +24,17 @@ ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [ + ent-id (inc (:topId db)) new-ts (next-ts db) ts-mp (last (:timestamped db)) - new-eavt (assoc (:EAVT ts-mp) (:e_id ent) (update-creation-ts ent new-ts) ) + fixed-ent (assoc ent :e_id ent-id) + new-eavt (assoc (:EAVT ts-mp) ent-id (update-creation-ts fixed-ent new-ts) ) new-avet (:AVET ts-mp) new-timesampedVal (assoc ts-mp :AVET new-avet :EAVT new-eavt )] (assoc db :timestamped (conj (:timestamped db) new-timesampedVal) - :curr-time new-ts ) + :curr-time new-ts + :topId ent-id) )) (defn make-db[] @@ -48,15 +51,15 @@ (def en1 (-> (make-entity "hotel") - (add-attr (make-attr "room" 12 :number)) - (add-attr (make-attr "address" "where" :string))) + (add-attr (make-attr :hotel/room 12 :number)) + (add-attr (make-attr :hotel/address "where" :string))) ) (def en2 (-> (make-entity "book") - (add-attr (make-attr "length" -1 :number)) - (add-attr (make-attr "auother" "jon" :string))) + (add-attr (make-attr :book/length -1 :number)) + (add-attr (make-attr :book/author "jon" :string))) ) From 0f9155fbc3517471b0ee8541e3dc2b7dff2deb67 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 9 Feb 2014 15:01:38 +0200 Subject: [PATCH 008/102] adding the avet indexing --- functionalDB/src/core/fdb.clj | 34 +++++++++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index a127ad2f4..f5b7e1e98 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -22,6 +22,28 @@ ) +; avet is a map as follows: {attrName -> {REFed-ent-id -> [REFing-elems-ids]}} + +(defn add-ref-to-avet[ent avet attr] + (let [ + attr-name (:name attr) + attr-val (:value attr) + attr-name-map (avet attr-name) + attr-name-map (if attr-name-map attr-name-map {attr-name {attr-val []}}) + reffed-ent-vec (attr-name-map attr-val) + reffed-ent-vec (if reffed-ent-vec reffed-ent-vec []) + ] + + (assoc-in avet [attr-name attr-val] (conj reffed-ent-vec (:e_id ent)))) + +) + +(defn update-avet[old-avet ent] + (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) + add-ref (partial add-ref-to-avet ent)] + (reduce add-ref old-avet reffingAttrs)) + ) + ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [ ent-id (inc (:topId db)) @@ -29,16 +51,18 @@ ts-mp (last (:timestamped db)) fixed-ent (assoc ent :e_id ent-id) new-eavt (assoc (:EAVT ts-mp) ent-id (update-creation-ts fixed-ent new-ts) ) - new-avet (:AVET ts-mp) - new-timesampedVal (assoc ts-mp :AVET new-avet :EAVT new-eavt )] + new-avet (update-avet (:AVET ts-mp) fixed-ent) + new-indices (assoc ts-mp :AVET new-avet :EAVT new-eavt )] (assoc db - :timestamped (conj (:timestamped db) new-timesampedVal) + :timestamped (conj (:timestamped db) new-indices) :curr-time new-ts :topId ent-id) )) (defn make-db[] - (atom {:timestamped [{:EAVT {} :AVET {}}] + (atom {:timestamped [{:EAVT {} ; all the entity info + :AVET {} ; for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + }] :topId 0 :curr-time 0 } @@ -58,7 +82,7 @@ (def en2 (-> (make-entity "book") - (add-attr (make-attr :book/length -1 :number)) + (add-attr (make-attr :book/found-at 1 :REF)) (add-attr (make-attr :book/author "jon" :string))) ) From 274f4951bfb3c5061150a8c2300f931e84739f06 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 10 Feb 2014 10:49:44 +0200 Subject: [PATCH 009/102] changing avet to aevt indexing --- functionalDB/src/core/fdb.clj | 52 +++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 20 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index f5b7e1e98..771620b8d 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -17,42 +17,47 @@ updatedAttrs (interleave ks updatedAttrsVals) ] (assoc ent :attrs updatedAttrs) - ) ) -; avet is a map as follows: {attrName -> {REFed-ent-id -> [REFing-elems-ids]}} -(defn add-ref-to-avet[ent avet attr] - (let [ - attr-name (:name attr) - attr-val (:value attr) - attr-name-map (avet attr-name) - attr-name-map (if attr-name-map attr-name-map {attr-name {attr-val []}}) - reffed-ent-vec (attr-name-map attr-val) - reffed-ent-vec (if reffed-ent-vec reffed-ent-vec []) - ] - (assoc-in avet [attr-name attr-val] (conj reffed-ent-vec (:e_id ent)))) +; AEVT -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} +; this basically provides the info - for each entity that is REFFed by others, who are the others who are REFing it, separated +; by the names of the attribute used for reffing + +(defn add-ref-to-aevt[ent aevt attr] + (let [ + reffed-id (:value attr) + attr-name (:name attr) + reffed-ent-map (aevt reffed-id) + reffed-ent-map (if reffed-ent-map reffed-ent-map {reffed-id {attr-name []}}) + reffing-ents-vec (attr-name reffed-ent-map) + reffing-ents-vec (if reffing-ents-vec reffing-ents-vec []) + ] + (assoc aevt reffed-id (assoc reffed-ent-map attr-name (conj reffing-ents-vec (:e_id ent)))) + ) ) -(defn update-avet[old-avet ent] +(defn update-aevt[old-aevt ent] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) - add-ref (partial add-ref-to-avet ent)] - (reduce add-ref old-avet reffingAttrs)) + add-ref (partial add-ref-to-aevt ent)] + (reduce add-ref old-aevt reffingAttrs)) ) ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [ + ent-id (inc (:topId db)) new-ts (next-ts db) ts-mp (last (:timestamped db)) fixed-ent (assoc ent :e_id ent-id) new-eavt (assoc (:EAVT ts-mp) ent-id (update-creation-ts fixed-ent new-ts) ) - new-avet (update-avet (:AVET ts-mp) fixed-ent) - new-indices (assoc ts-mp :AVET new-avet :EAVT new-eavt )] + old-aevt (:AEVT ts-mp) + new-aevt (update-aevt old-aevt fixed-ent) + new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt )] (assoc db :timestamped (conj (:timestamped db) new-indices) :curr-time new-ts @@ -61,7 +66,7 @@ (defn make-db[] (atom {:timestamped [{:EAVT {} ; all the entity info - :AVET {} ; for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + :AEVT {} ; for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) }] :topId 0 :curr-time 0 @@ -87,14 +92,21 @@ ) -(swap! db1 add-entity en1) +(def en3 (-> (make-entity "gate") -(swap! db1 add-entity en2) + (add-attr (make-attr :book/found-at 1 :REF)) + (add-attr (make-attr :gate/color "black" :string))) + ) +(swap! db1 add-entity en1) + +(swap! db1 add-entity en2) +(swap! db1 add-entity en3) +(:AEVT (last (:timestamped @db1))) ;(defn recent-ts-val [db](last (:timestamped db))) From a9d8e57751105bec47e5b88f7794cc5b333eb03d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 10 Feb 2014 13:04:22 +0200 Subject: [PATCH 010/102] creation of REFed attributes done with the REDed entity and not its id --- functionalDB/src/core/fdb.clj | 50 ++++++++++------------------------- 1 file changed, 14 insertions(+), 36 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 771620b8d..85d76cc78 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -3,76 +3,54 @@ (defrecord Entity [e_id name attrs]) (defrecord Attr [name type value ts prev-ts]) - (defn make-entity [name] (Entity. :no-id-yet name {})) -(defn make-attr[name val type] (Attr. name type val -1 -1)) +(defn make-attr[name val type] (Attr. name type (if (= :REF type) (:ent_id val) val) -1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) - (defn update-creation-ts [ent tsVal] (let [ks (keys (:attrs ent)) vls (vals (:attrs ent)) updatedAttrsVals (map #(assoc % :ts tsVal) vls) updatedAttrs (interleave ks updatedAttrsVals) - ] - (assoc ent :attrs updatedAttrs) - ) - - ) - - + ](assoc ent :attrs updatedAttrs))) ; AEVT -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} ; this basically provides the info - for each entity that is REFFed by others, who are the others who are REFing it, separated ; by the names of the attribute used for reffing - - (defn add-ref-to-aevt[ent aevt attr] - (let [ - reffed-id (:value attr) + (let [reffed-id (:value attr) attr-name (:name attr) reffed-ent-map (aevt reffed-id) reffed-ent-map (if reffed-ent-map reffed-ent-map {reffed-id {attr-name []}}) reffing-ents-vec (attr-name reffed-ent-map) reffing-ents-vec (if reffing-ents-vec reffing-ents-vec []) - ] - (assoc aevt reffed-id (assoc reffed-ent-map attr-name (conj reffing-ents-vec (:e_id ent)))) - ) -) + ] (assoc aevt reffed-id (assoc reffed-ent-map attr-name (conj reffing-ents-vec (:e_id ent)))) + )) (defn update-aevt[old-aevt ent] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) add-ref (partial add-ref-to-aevt ent)] - (reduce add-ref old-aevt reffingAttrs)) - ) + (reduce add-ref old-aevt reffingAttrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity[db ent] (let [ - - ent-id (inc (:topId db)) +(defn add-entity[db ent] (let [ent-id (inc (:topId db)) new-ts (next-ts db) ts-mp (last (:timestamped db)) fixed-ent (assoc ent :e_id ent-id) new-eavt (assoc (:EAVT ts-mp) ent-id (update-creation-ts fixed-ent new-ts) ) old-aevt (:AEVT ts-mp) new-aevt (update-aevt old-aevt fixed-ent) - new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt )] - (assoc db + new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt ) + ](assoc db :timestamped (conj (:timestamped db) new-indices) :curr-time new-ts - :topId ent-id) - )) + :topId ent-id))) (defn make-db[] (atom {:timestamped [{:EAVT {} ; all the entity info - :AEVT {} ; for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) - }] - :topId 0 - :curr-time 0 - } - ) - ) + :AEVT {}}] ; for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + :topId 0 :curr-time 0})) @@ -87,14 +65,14 @@ (def en2 (-> (make-entity "book") - (add-attr (make-attr :book/found-at 1 :REF)) + (add-attr (make-attr :book/found-at en1 :REF)) (add-attr (make-attr :book/author "jon" :string))) ) (def en3 (-> (make-entity "gate") - (add-attr (make-attr :book/found-at 1 :REF)) + (add-attr (make-attr :book/found-at en1 :REF)) (add-attr (make-attr :gate/color "black" :string))) ) From e4437dcd6caa1d149535b7e0b9a1ac42cdb39c74 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 10 Feb 2014 15:52:47 +0200 Subject: [PATCH 011/102] reffing entities in AEVT are now held in a set and not in a vector --- functionalDB/src/core/fdb.clj | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 85d76cc78..ef470df44 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -4,7 +4,7 @@ (defrecord Attr [name type value ts prev-ts]) (defn make-entity [name] (Entity. :no-id-yet name {})) -(defn make-attr[name val type] (Attr. name type (if (= :REF type) (:ent_id val) val) -1 -1)) +(defn make-attr[name val type] (Attr. name type (if (= :REF type) (:e_id val) val)-1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) @@ -21,11 +21,8 @@ (defn add-ref-to-aevt[ent aevt attr] (let [reffed-id (:value attr) attr-name (:name attr) - reffed-ent-map (aevt reffed-id) - reffed-ent-map (if reffed-ent-map reffed-ent-map {reffed-id {attr-name []}}) - reffing-ents-vec (attr-name reffed-ent-map) - reffing-ents-vec (if reffing-ents-vec reffing-ents-vec []) - ] (assoc aevt reffed-id (assoc reffed-ent-map attr-name (conj reffing-ents-vec (:e_id ent)))) + reffing-ent (get-in aevt [reffed-id attr-name] #{} ) + ] (assoc-in aevt [reffed-id attr-name] (conj reffing-ent (:e_id ent))) )) (defn update-aevt[old-aevt ent] @@ -33,12 +30,14 @@ add-ref (partial add-ref-to-aevt ent)] (reduce add-ref old-aevt reffingAttrs))) +(defn nextId[db] (let [topId (inc(:topId db))] [topId (keyword (str topId))])) + ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity[db ent] (let [ent-id (inc (:topId db)) +(defn add-entity[db ent] (let [[ent-id ent-id-key ] (nextId db) new-ts (next-ts db) ts-mp (last (:timestamped db)) - fixed-ent (assoc ent :e_id ent-id) - new-eavt (assoc (:EAVT ts-mp) ent-id (update-creation-ts fixed-ent new-ts) ) + fixed-ent (assoc ent :e_id ent-id-key) + new-eavt (assoc (:EAVT ts-mp) ent-id-key (update-creation-ts fixed-ent new-ts) ) old-aevt (:AEVT ts-mp) new-aevt (update-aevt old-aevt fixed-ent) new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt ) @@ -63,28 +62,33 @@ ) +(swap! db1 add-entity en1) +(def ref1 (:1 (:EAVT(last (:timestamped @db1))))) + (def en2 (-> (make-entity "book") - (add-attr (make-attr :book/found-at en1 :REF)) - (add-attr (make-attr :book/author "jon" :string))) + ; (add-attr (make-attr :book/author "jon" :string)) + (add-attr (make-attr :book/found-at ref1 :REF))) ) (def en3 (-> (make-entity "gate") - - (add-attr (make-attr :book/found-at en1 :REF)) - (add-attr (make-attr :gate/color "black" :string))) + (add-attr (make-attr :gate/color "black" :string)) + (add-attr (make-attr :book/found-at ref1 :REF)) + ) ) -(swap! db1 add-entity en1) + + + (swap! db1 add-entity en2) (swap! db1 add-entity en3) -(:AEVT (last (:timestamped @db1))) + ;(defn recent-ts-val [db](last (:timestamped db))) From 740e2fd4d3f14e14a50139ce3cc56e270b938db2 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 10 Feb 2014 17:34:45 +0200 Subject: [PATCH 012/102] clients can provide Id for entities, when no id is provided, a running counter value is used as an id --- functionalDB/src/core/fdb.clj | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index ef470df44..d27c17512 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -3,7 +3,10 @@ (defrecord Entity [e_id name attrs]) (defrecord Attr [name type value ts prev-ts]) -(defn make-entity [name] (Entity. :no-id-yet name {})) +(defn make-entity + ([name] (make-entity :no-id-yet name)) + ([name id] (Entity. id name {}))) + (defn make-attr[name val type] (Attr. name type (if (= :REF type) (:e_id val) val)-1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) @@ -30,10 +33,16 @@ add-ref (partial add-ref-to-aevt ent)] (reduce add-ref old-aevt reffingAttrs))) -(defn nextId[db] (let [topId (inc(:topId db))] [topId (keyword (str topId))])) +(defn nextId[db ent] ( + let [ topId (:topId db) + entId (:e_id ent) + [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] + [idToUse (keyword (str idToUse)) nextTop]) + + ) ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity[db ent] (let [[ent-id ent-id-key ] (nextId db) +(defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) new-ts (next-ts db) ts-mp (last (:timestamped db)) fixed-ent (assoc ent :e_id ent-id-key) @@ -44,18 +53,17 @@ ](assoc db :timestamped (conj (:timestamped db) new-indices) :curr-time new-ts - :topId ent-id))) + :topId next-top))) (defn make-db[] - (atom {:timestamped [{:EAVT {} ; all the entity info - :AEVT {}}] ; for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + (atom {:timestamped [{:EAVT {} :AEVT {}}]; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) :topId 0 :curr-time 0})) (def db1 (make-db)) -(def en1 (-> (make-entity "hotel") +(def en1 (-> (make-entity "hotel" "hiilt" ) (add-attr (make-attr :hotel/room 12 :number)) (add-attr (make-attr :hotel/address "where" :string))) @@ -63,20 +71,19 @@ ) (swap! db1 add-entity en1) -(def ref1 (:1 (:EAVT(last (:timestamped @db1))))) +(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) (def en2 (-> (make-entity "book") - ; (add-attr (make-attr :book/author "jon" :string)) + (add-attr (make-attr :book/author "jon" :string)) (add-attr (make-attr :book/found-at ref1 :REF))) ) (def en3 (-> (make-entity "gate") (add-attr (make-attr :gate/color "black" :string)) - (add-attr (make-attr :book/found-at ref1 :REF)) + (add-attr (make-attr :book/found-at ref1 :REF)) ) - ) From d10612cad9bd734f210900d3939e99a046e24e6b Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Feb 2014 14:17:38 +0200 Subject: [PATCH 013/102] adding transacting --- functionalDB/src/core/fdb.clj | 74 +++++++++++++++-------------------- 1 file changed, 31 insertions(+), 43 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index d27c17512..15ab757a8 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -3,8 +3,7 @@ (defrecord Entity [e_id name attrs]) (defrecord Attr [name type value ts prev-ts]) -(defn make-entity - ([name] (make-entity :no-id-yet name)) +(defn make-entity ([name] (make-entity :no-id-yet name)) ([name id] (Entity. id name {}))) (defn make-attr[name val type] (Attr. name type (if (= :REF type) (:e_id val) val)-1 -1)) @@ -37,9 +36,7 @@ let [ topId (:topId db) entId (:e_id ent) [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] - [idToUse (keyword (str idToUse)) nextTop]) - - ) + [idToUse (keyword (str idToUse)) nextTop])) ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) @@ -52,15 +49,29 @@ new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt ) ](assoc db :timestamped (conj (:timestamped db) new-indices) - :curr-time new-ts :topId next-top))) -(defn make-db[] - (atom {:timestamped [{:EAVT {} :AEVT {}}]; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) +(defn db-from-transacted [initial txs] + (loop [[tx & rst-tx] txs transacted initial] + (if tx + (recur rst-tx ((first tx) transacted (second tx))) + (let [new-ts (next-ts initial) + initial-indices (:timestamped initial ) + new-indices (last (:timestamped transacted)) + new-topId (:topId transacted)] + (assoc initial :timestamped (conj initial-indices new-indices) :curr-time new-ts :topId new-topId))))) + +(defmacro transact [db & txs] + (when txs + (loop [[frst-tx# & rst-tx#] txs res# ['swap! db 'db-from-transacted] cnt# []] + (if frst-tx# + (recur rst-tx# res# (conj cnt# (vec frst-tx#))) + (list* (conj res# cnt#)))))) + +(defn make-db[] ; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + (atom {:timestamped [{:EAVT {} :AEVT {}}] :topId 0 :curr-time 0})) - - (def db1 (make-db)) (def en1 (-> (make-entity "hotel" "hiilt" ) @@ -69,52 +80,29 @@ (add-attr (make-attr :hotel/address "where" :string))) ) - -(swap! db1 add-entity en1) (def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) (def en2 (-> (make-entity "book") - - (add-attr (make-attr :book/author "jon" :string)) - (add-attr (make-attr :book/found-at ref1 :REF))) - ) - -(def en3 (-> (make-entity "gate") - (add-attr (make-attr :gate/color "black" :string)) - (add-attr (make-attr :book/found-at ref1 :REF)) - ) - ) - - - - - -(swap! db1 add-entity en2) - + (add-attr (make-attr :book/found-at ref1 :REF)))) -(swap! db1 add-entity en3) +(transact db1 (add-entity en1) (add-entity en2)) +(macroexpand-1 '(transact2 db1 (add-entity en1) (add-entity en2))) +;(swap! db1 add-entity en1) +;(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) -;(defn recent-ts-val [db](last (:timestamped db))) -;(defn update-ts-with-EAV [ts &[e a v] :as more] -; (let[eavt (:EAVT ts)] -; (assoc-in ts :EAVT e a v) -; ) -; ) +;; (def en3 (-> (make-entity "gate") +;; (add-attr (make-attr :gate/color "black" :string)) +;; (add-attr (make-attr :book/found-at ref1 :REF)) )) -;(defn _add-timestamp[db & more] -; (let [ ts {:EAVT (:EAVT (recent-ts-val db))}] -; (update-ts-with-EAV ts more) -; -; (assoc db :timestamped (conj (:timestamped db) ts)) -; ) -; ) +;(swap! db1 add-entity en2) +;(swap! db1 add-entity en3) ;(add-attr @db1 "name" :string "Jim" ) From 774e70e746d6fcc63e96dfeaab750ec1e2259761 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Feb 2014 14:36:28 +0200 Subject: [PATCH 014/102] clearer transacting --- functionalDB/src/core/fdb.clj | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 15ab757a8..2bc98b89b 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -53,13 +53,12 @@ (defn db-from-transacted [initial txs] (loop [[tx & rst-tx] txs transacted initial] - (if tx - (recur rst-tx ((first tx) transacted (second tx))) - (let [new-ts (next-ts initial) - initial-indices (:timestamped initial ) - new-indices (last (:timestamped transacted)) - new-topId (:topId transacted)] - (assoc initial :timestamped (conj initial-indices new-indices) :curr-time new-ts :topId new-topId))))) + (if tx (recur rst-tx ((first tx) transacted (second tx))) + (let [ initial-indices (:timestamped initial ) + new-indices (last (:timestamped transacted))] + (assoc initial :timestamped (conj initial-indices new-indices) + :curr-time (next-ts initial) + :topId (:topId transacted)))))) (defmacro transact [db & txs] (when txs From 155de764895e4db0c74ef57fae76fcc2717cc2d3 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Feb 2014 14:48:39 +0200 Subject: [PATCH 015/102] transation would work now for fns that have several arguments --- functionalDB/src/core/fdb.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 2bc98b89b..3b2cd8074 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -53,7 +53,7 @@ (defn db-from-transacted [initial txs] (loop [[tx & rst-tx] txs transacted initial] - (if tx (recur rst-tx ((first tx) transacted (second tx))) + (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [ initial-indices (:timestamped initial ) new-indices (last (:timestamped transacted))] (assoc initial :timestamped (conj initial-indices new-indices) From bedb6c247b799ed498c9f979fced5261a9b7703f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Feb 2014 19:10:12 +0200 Subject: [PATCH 016/102] adding removal of entity --- functionalDB/src/core/fdb.clj | 98 ++++++++++++++++++++++++----------- 1 file changed, 69 insertions(+), 29 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 3b2cd8074..9ff874333 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -3,18 +3,28 @@ (defrecord Entity [e_id name attrs]) (defrecord Attr [name type value ts prev-ts]) +(defn make-db[] ; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + (atom {:timestamped [{:EAVT {} :AEVT {}}] + :topId 0 :curr-time 0})) + (defn make-entity ([name] (make-entity :no-id-yet name)) ([name id] (Entity. id name {}))) (defn make-attr[name val type] (Attr. name type (if (= :REF type) (:e_id val) val)-1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) + (defn next-ts [db] (inc (:curr-time db))) +(defn nextId[db ent] ( + let [ topId (:topId db) + entId (:e_id ent) + [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] + [idToUse (keyword (str idToUse)) nextTop])) (defn update-creation-ts [ent tsVal] (let [ks (keys (:attrs ent)) vls (vals (:attrs ent)) updatedAttrsVals (map #(assoc % :ts tsVal) vls) - updatedAttrs (interleave ks updatedAttrsVals) + updatedAttrs (zipmap ks updatedAttrsVals) ](assoc ent :attrs updatedAttrs))) ; AEVT -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} @@ -23,21 +33,16 @@ (defn add-ref-to-aevt[ent aevt attr] (let [reffed-id (:value attr) attr-name (:name attr) - reffing-ent (get-in aevt [reffed-id attr-name] #{} ) - ] (assoc-in aevt [reffed-id attr-name] (conj reffing-ent (:e_id ent))) - )) + back-reffing-set (get-in aevt [reffed-id attr-name] #{} ) + new-back-reffing-set (conj back-reffing-set (:e_id ent)) + ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set) + )) (defn update-aevt[old-aevt ent] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) add-ref (partial add-ref-to-aevt ent)] (reduce add-ref old-aevt reffingAttrs))) -(defn nextId[db ent] ( - let [ topId (:topId db) - entId (:e_id ent) - [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] - [idToUse (keyword (str idToUse)) nextTop])) - ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) new-ts (next-ts db) @@ -51,52 +56,87 @@ :timestamped (conj (:timestamped db) new-indices) :topId next-top))) -(defn db-from-transacted [initial txs] - (loop [[tx & rst-tx] txs transacted initial] +(defn remove-ref-from-aevt[ent aevt attr] + (let [reffed-id (:value attr) + attr-name (:name attr) + back-reffing-set (get-in aevt [reffed-id attr-name]) + new-back-reffing-set (disj back-reffing-set (:e_id ent)) + ] + (assoc-in aevt [reffed-id attr-name] new-back-reffing-set)) + ) + +(defn remove-outgoing-refs [ent aevt] + (let [ reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) + remove-ref (partial remove-ref-from-aevt ent)] + (reduce remove-ref aevt reffingAttrs) + ) + ) + +(defn remove-entity[db ent] + (let [ent-id (:e_id ent) + indices (last (:timestamped db)) + eavt (:EAVT indices) + aevt (:AEVT indices) + aevt (remove-outgoing-refs ent aevt) + aevt aevt + new-eavt (dissoc eavt ent-id) ; removing the entity + new-aevt (dissoc aevt ent-id) ; removing incoming REFs to the entity + new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt) + ] + (assoc db :timestamped new-indices))) + +(defn transact-on-db [initial-db txs] + (loop [[tx & rst-tx] txs transacted initial-db] (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) - (let [ initial-indices (:timestamped initial ) + (let [ initial-indices (:timestamped initial-db) new-indices (last (:timestamped transacted))] - (assoc initial :timestamped (conj initial-indices new-indices) - :curr-time (next-ts initial) + (assoc initial-db :timestamped (conj initial-indices new-indices) + :curr-time (next-ts initial-db) :topId (:topId transacted)))))) (defmacro transact [db & txs] (when txs - (loop [[frst-tx# & rst-tx#] txs res# ['swap! db 'db-from-transacted] cnt# []] - (if frst-tx# - (recur rst-tx# res# (conj cnt# (vec frst-tx#))) - (list* (conj res# cnt#)))))) + (loop [[frst-tx# & rst-tx#] txs res# ['swap! db 'transact-on-db] cnt# []] + (if frst-tx# (recur rst-tx# res# (conj cnt# (vec frst-tx#))) + (list* (conj res# cnt#)))))) + -(defn make-db[] ; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) - (atom {:timestamped [{:EAVT {} :AEVT {}}] - :topId 0 :curr-time 0})) (def db1 (make-db)) (def en1 (-> (make-entity "hotel" "hiilt" ) - (add-attr (make-attr :hotel/room 12 :number)) + (add-attr (make-attr :hotel/room 12 :number)) (add-attr (make-attr :hotel/address "where" :string))) ) -(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) +(transact db1 (add-entity en1)) + +(def rel-e1 (get-in (last (:timestamped @db1)) [:EAVT :hiilt])) +(transact db1 (remove-entity rel-e1)) +;(add-entity @db1 en1) + +(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) +;(type(:attrs ref1)) (def en2 (-> (make-entity "book") (add-attr (make-attr :book/author "jon" :string)) (add-attr (make-attr :book/found-at ref1 :REF)))) + (def en3 (-> (make-entity "gate") + (add-attr (make-attr :gate/color "black" :string)) + (add-attr (make-attr :book/found-at ref1 :REF)) )) -(transact db1 (add-entity en1) (add-entity en2)) +;(transact db1 (add-entity en2) (add-entity en3)) -(macroexpand-1 '(transact2 db1 (add-entity en1) (add-entity en2))) +;(remove-entity @db1 rel-e1) +;(macroexpand-1 '(transact2 db1 (add-entity en1) (add-entity en2))) ;(swap! db1 add-entity en1) ;(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) -;; (def en3 (-> (make-entity "gate") -;; (add-attr (make-attr :gate/color "black" :string)) -;; (add-attr (make-attr :book/found-at ref1 :REF)) )) + ;(swap! db1 add-entity en2) From 6422e995b53529c6e386513a2bdb32db32d53705 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Feb 2014 19:10:42 +0200 Subject: [PATCH 017/102] removal of entity - cleanup --- functionalDB/src/core/fdb.clj | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 9ff874333..1dfb04fec 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -60,29 +60,21 @@ (let [reffed-id (:value attr) attr-name (:name attr) back-reffing-set (get-in aevt [reffed-id attr-name]) - new-back-reffing-set (disj back-reffing-set (:e_id ent)) - ] - (assoc-in aevt [reffed-id attr-name] new-back-reffing-set)) - ) + new-back-reffing-set (disj back-reffing-set (:e_id ent))] + (assoc-in aevt [reffed-id attr-name] new-back-reffing-set))) (defn remove-outgoing-refs [ent aevt] (let [ reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) remove-ref (partial remove-ref-from-aevt ent)] - (reduce remove-ref aevt reffingAttrs) - ) - ) + (reduce remove-ref aevt reffingAttrs))) (defn remove-entity[db ent] (let [ent-id (:e_id ent) indices (last (:timestamped db)) - eavt (:EAVT indices) - aevt (:AEVT indices) - aevt (remove-outgoing-refs ent aevt) - aevt aevt - new-eavt (dissoc eavt ent-id) ; removing the entity + aevt (remove-outgoing-refs ent (:AEVT indices)) + new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity new-aevt (dissoc aevt ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt) - ] + new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt) ] (assoc db :timestamped new-indices))) (defn transact-on-db [initial-db txs] From 4bf3e4dc329b06db478ab04788cf680f8fff17e8 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Feb 2014 22:35:44 +0200 Subject: [PATCH 018/102] add / remove entity are now arguemnted HOF update-entity bug fix in creating entity --- functionalDB/src/core/fdb.clj | 59 ++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 1dfb04fec..a75841928 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -8,7 +8,7 @@ :topId 0 :curr-time 0})) (defn make-entity ([name] (make-entity :no-id-yet name)) - ([name id] (Entity. id name {}))) + ([id name] (Entity. id name {}))) (defn make-attr[name val type] (Attr. name type (if (= :REF type) (:e_id val) val)-1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) @@ -30,17 +30,17 @@ ; AEVT -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} ; this basically provides the info - for each entity that is REFFed by others, who are the others who are REFing it, separated ; by the names of the attribute used for reffing -(defn add-ref-to-aevt[ent aevt attr] +(defn add-ref-to-aevt[ent operation aevt attr] (let [reffed-id (:value attr) attr-name (:name attr) back-reffing-set (get-in aevt [reffed-id attr-name] #{} ) - new-back-reffing-set (conj back-reffing-set (:e_id ent)) + new-back-reffing-set (operation back-reffing-set (:e_id ent)) ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set) )) -(defn update-aevt[old-aevt ent] +(defn update-aevt[old-aevt ent operation] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) - add-ref (partial add-ref-to-aevt ent)] + add-ref (partial add-ref-to-aevt ent operation)] (reduce add-ref old-aevt reffingAttrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one @@ -50,41 +50,32 @@ fixed-ent (assoc ent :e_id ent-id-key) new-eavt (assoc (:EAVT ts-mp) ent-id-key (update-creation-ts fixed-ent new-ts) ) old-aevt (:AEVT ts-mp) - new-aevt (update-aevt old-aevt fixed-ent) + new-aevt (update-aevt old-aevt fixed-ent conj) new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt ) ](assoc db :timestamped (conj (:timestamped db) new-indices) :topId next-top))) -(defn remove-ref-from-aevt[ent aevt attr] - (let [reffed-id (:value attr) - attr-name (:name attr) - back-reffing-set (get-in aevt [reffed-id attr-name]) - new-back-reffing-set (disj back-reffing-set (:e_id ent))] - (assoc-in aevt [reffed-id attr-name] new-back-reffing-set))) - -(defn remove-outgoing-refs [ent aevt] - (let [ reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) - remove-ref (partial remove-ref-from-aevt ent)] - (reduce remove-ref aevt reffingAttrs))) - (defn remove-entity[db ent] (let [ent-id (:e_id ent) indices (last (:timestamped db)) - aevt (remove-outgoing-refs ent (:AEVT indices)) + aevt (update-aevt (:AEVT indices) ent disj) new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity new-aevt (dissoc aevt ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt) ] - (assoc db :timestamped new-indices))) + new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt) + res (assoc db :timestamped (conj (:timestamped db) new-indices))] + res)) (defn transact-on-db [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) - (let [ initial-indices (:timestamped initial-db) - new-indices (last (:timestamped transacted))] - (assoc initial-db :timestamped (conj initial-indices new-indices) + (let [ initial-indices (:timestamped initial-db) + new-indices (last (:timestamped transacted)) + transacted transacted + res (assoc initial-db :timestamped (conj initial-indices new-indices) :curr-time (next-ts initial-db) - :topId (:topId transacted)))))) + :topId (:topId transacted)) ] + res)))) (defmacro transact [db & txs] (when txs @@ -96,7 +87,7 @@ (def db1 (make-db)) -(def en1 (-> (make-entity "hotel" "hiilt" ) +(def en1 (-> (make-entity "hiilt" "hotel" ) (add-attr (make-attr :hotel/room 12 :number)) (add-attr (make-attr :hotel/address "where" :string))) @@ -106,7 +97,9 @@ (def rel-e1 (get-in (last (:timestamped @db1)) [:EAVT :hiilt])) -(transact db1 (remove-entity rel-e1)) + + + ;(add-entity @db1 en1) (def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) @@ -117,11 +110,19 @@ (def en3 (-> (make-entity "gate") (add-attr (make-attr :gate/color "black" :string)) - (add-attr (make-attr :book/found-at ref1 :REF)) )) + (add-attr (make-attr :gate/found-at ref1 :REF)) )) +(transact db1 (add-entity en2) (add-entity en3)) + + + ;(transact db1 (remove-entity ref1)) + + ;(transact db1 (add-entity en2) (add-entity en3)) -;(remove-entity @db1 rel-e1) + +;(swap! db1 transact-on-db [[remove-entity ref1]]) +;(macroexpand-1 '(transact db1 (remove-entity ref1))) ;(macroexpand-1 '(transact2 db1 (add-entity en1) (add-entity en2))) ;(swap! db1 add-entity en1) ;(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) From fdb1e0de98fcf5c7b1d9c22ee0b34395307796c1 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Thu, 13 Feb 2014 14:03:20 +0200 Subject: [PATCH 019/102] adding the ability to update a datum --- functionalDB/src/core/fdb.clj | 51 ++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index a75841928..fb3a68b79 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -10,7 +10,7 @@ (defn make-entity ([name] (make-entity :no-id-yet name)) ([id name] (Entity. id name {}))) -(defn make-attr[name val type] (Attr. name type (if (= :REF type) (:e_id val) val)-1 -1)) +(defn make-attr[name val type] (Attr. name type (val-from-ref type val) -1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) @@ -20,6 +20,8 @@ [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] [idToUse (keyword (str idToUse)) nextTop])) +(defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:e_id attr-val) attr-val )) + (defn update-creation-ts [ent tsVal] (let [ks (keys (:attrs ent)) vls (vals (:attrs ent)) @@ -46,12 +48,11 @@ ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) new-ts (next-ts db) - ts-mp (last (:timestamped db)) + indices (last (:timestamped db)) fixed-ent (assoc ent :e_id ent-id-key) - new-eavt (assoc (:EAVT ts-mp) ent-id-key (update-creation-ts fixed-ent new-ts) ) - old-aevt (:AEVT ts-mp) - new-aevt (update-aevt old-aevt fixed-ent conj) - new-indices (assoc ts-mp :AEVT new-aevt :EAVT new-eavt ) + new-eavt (assoc (:EAVT indices) ent-id-key (update-creation-ts fixed-ent new-ts) ) + new-aevt (update-aevt (:AEVT indices) fixed-ent conj) + new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt ) ](assoc db :timestamped (conj (:timestamped db) new-indices) :topId next-top))) @@ -83,19 +84,38 @@ (if frst-tx# (recur rst-tx# res# (conj cnt# (vec frst-tx#))) (list* (conj res# cnt#)))))) +(defn update-aevt-for-datum [aevt ent-id attr new-val] + (if (not= :REF (:type attr )) + aevt + (let [ old-ref-id (:value attr) + attr-name (:name attr) + old-reffed (get-in aevt [old-ref-id attr-name]) + cleaned-aevt (assoc-in aevt [old-ref-id attr-name] (disj old-reffed ent-id)) + to-be-updated-ref (get-in aevt [new-val attr-name] #{}) + updated-aevt (assoc-in cleaned-aevt [new-val attr-name] (conj to-be-updated-ref ent-id) ) + ] updated-aevt))) + +(defn update-datum [db ent-id att-name new-val] + (let [ new-ts (next-ts db) + indices (last (:timestamped db)) + attr (get-in indices [:EAVT ent-id :attrs att-name] ) + real-new-val (val-from-ref (:type attr) new-val) + updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) + eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) + new-aevt (update-aevt-for-datum (:AEVT indices) ent-id attr real-new-val) + fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt) + new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) + ]new-db)) (def db1 (make-db)) (def en1 (-> (make-entity "hiilt" "hotel" ) - (add-attr (make-attr :hotel/room 12 :number)) - (add-attr (make-attr :hotel/address "where" :string))) - - ) + (add-attr (make-attr :hotel/address "where" :string)))) (transact db1 (add-entity en1)) -(def rel-e1 (get-in (last (:timestamped @db1)) [:EAVT :hiilt])) +;(def rel-e1 (get-in (last (:timestamped @db1)) [:EAVT :hiilt])) @@ -103,6 +123,7 @@ ;(add-entity @db1 en1) (def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) +(:e_id ref1) ;(type(:attrs ref1)) (def en2 (-> (make-entity "book") (add-attr (make-attr :book/author "jon" :string)) @@ -111,14 +132,18 @@ (def en3 (-> (make-entity "gate") (add-attr (make-attr :gate/color "black" :string)) (add-attr (make-attr :gate/found-at ref1 :REF)) )) -(transact db1 (add-entity en2) (add-entity en3)) +;(transact db1 (add-entity en2) (add-entity en3)) ;(transact db1 (remove-entity ref1)) -;(transact db1 (add-entity en2) (add-entity en3)) +(transact db1 (add-entity en2) (add-entity en3)) +(def ref12 (:1 (:EAVT(last (:timestamped @db1))))) + + +(transact db1 (update-datum :2 :gate/found-at ref12 )) ;(swap! db1 transact-on-db [[remove-entity ref1]]) From ace5e920af834cd4592a007be321456facba8928 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Thu, 13 Feb 2014 17:15:10 +0200 Subject: [PATCH 020/102] updated description --- functionalDB/README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/functionalDB/README.md b/functionalDB/README.md index c93db369d..3f9139e0e 100644 --- a/functionalDB/README.md +++ b/functionalDB/README.md @@ -1,5 +1,11 @@ -# Jsomic +# FDB -Jsomic is an in-memory, no-sql functional database, written in Javascript. +FDB (TODO - fidn a better name) is an in-memory, no-sql functional database, written in Clojure. + +It is a modest attempt to provide part of the functionality that the Datomic database provides (the main omitted functionality is the durability part). + +The core idea is to add the notion of time to the data, and provide on top of the standard CRUD operations, a set of time related operations. + +The database itself is a collection of datums. A datum is built of an entity (think of a row in a table)that has its attributes (a column) and the value of that entity's attribute at a given time. +Any update does not overwrite the prvious value, but addes another datum to the database. -It is a modest attempt to provide part of the functionality that the Datomic database provides (the main omitted functionality is the durability part). \ No newline at end of file From fcb90508d87d1b9f6092b18a7de779ae76e7e34c Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 14 Feb 2014 13:24:21 +0200 Subject: [PATCH 021/102] Adding several different read operations, refactoring out test code --- functionalDB/src/core/fdb.clj | 99 +++++++++-------------------------- 1 file changed, 25 insertions(+), 74 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index fb3a68b79..a5433e5c1 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -84,7 +84,7 @@ (if frst-tx# (recur rst-tx# res# (conj cnt# (vec frst-tx#))) (list* (conj res# cnt#)))))) -(defn update-aevt-for-datum [aevt ent-id attr new-val] +(defn update-aevt-for-datom [aevt ent-id attr new-val] (if (not= :REF (:type attr )) aevt (let [ old-ref-id (:value attr) @@ -95,86 +95,37 @@ updated-aevt (assoc-in cleaned-aevt [new-val attr-name] (conj to-be-updated-ref ent-id) ) ] updated-aevt))) -(defn update-datum [db ent-id att-name new-val] +(defn update-datom [db ent-id att-name new-val] (let [ new-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name] ) real-new-val (val-from-ref (:type attr) new-val) updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) - new-aevt (update-aevt-for-datum (:AEVT indices) ent-id attr real-new-val) + new-aevt (update-aevt-for-datom (:AEVT indices) ent-id attr real-new-val) fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt) new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) ]new-db)) - -(def db1 (make-db)) - -(def en1 (-> (make-entity "hiilt" "hotel" ) - (add-attr (make-attr :hotel/room 12 :number)) - (add-attr (make-attr :hotel/address "where" :string)))) -(transact db1 (add-entity en1)) - -;(def rel-e1 (get-in (last (:timestamped @db1)) [:EAVT :hiilt])) - - - - -;(add-entity @db1 en1) - -(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) -(:e_id ref1) -;(type(:attrs ref1)) -(def en2 (-> (make-entity "book") - (add-attr (make-attr :book/author "jon" :string)) - (add-attr (make-attr :book/found-at ref1 :REF)))) - - (def en3 (-> (make-entity "gate") - (add-attr (make-attr :gate/color "black" :string)) - (add-attr (make-attr :gate/found-at ref1 :REF)) )) -;(transact db1 (add-entity en2) (add-entity en3)) - - - ;(transact db1 (remove-entity ref1)) - - - -(transact db1 (add-entity en2) (add-entity en3)) -(def ref12 (:1 (:EAVT(last (:timestamped @db1))))) - - -(transact db1 (update-datum :2 :gate/found-at ref12 )) - - -;(swap! db1 transact-on-db [[remove-entity ref1]]) -;(macroexpand-1 '(transact db1 (remove-entity ref1))) -;(macroexpand-1 '(transact2 db1 (add-entity en1) (add-entity en2))) -;(swap! db1 add-entity en1) -;(def ref1 (:hiilt (:EAVT(last (:timestamped @db1))))) - - - - - -;(swap! db1 add-entity en2) - - -;(swap! db1 add-entity en3) - - - ;(add-attr @db1 "name" :string "Jim" ) - ; (add-attr @db1 "sur-name" :string "Doe" ) - ; (add-attr @db1 "age-name" :number 39 ) - - -;db -;; (defn fact[n] -;; (if (<= n 0) 1 -;; (* n (fact (dec n))) -;; ) - -;; ) - -;; (fact 1) - -;(swap! db assoc :6 5 +(defn attr-at "The attribute of an entity at a given time (defaults to recent time)" + ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name ts] + (let [indices ((:timestamped db) ts)] (get-in indices [:EAVT ent-id :attrs attr-name])))) + +(defn value-of-at "value of a datom at a given time, if no time is provided, we default to the most recent value" + ([db e-id attr-name] (:value (attr-at db e-id attr-name))) + ([db e-id attr-name ts] (:value (attr-at db e-id attr-name ts)))) + +(defn relates-to-as "returns a seq of all the entities that had an attribute named attr-name whose type is REF and the value was e-id, all this at a given time" + ([db e-id attr-name] (relates-to-as db e-id attr-name (:curr-time db))) + ([db e-id attr-name ts] + (let [indices ((:timestamped db) ts) + reffing-ids (get-in indices [:AEVT e-id attr-name]) + ] + (map #(get-in indices [:EAVT %]) reffing-ids )))) + +(defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] + (loop [res [] ts (:curr-time db)] + (if (= -1 ts) (reverse res) + (let [attr (attr-at db ent-id attr-name ts)] + (recur (conj res {ts (:value attr)}) (:prev-ts attr)))))) From 3e822f959f87617e392b56fe9a97e85292e14c57 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 14 Feb 2014 13:25:55 +0200 Subject: [PATCH 022/102] fixing typo --- functionalDB/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/README.md b/functionalDB/README.md index 3f9139e0e..e659681f8 100644 --- a/functionalDB/README.md +++ b/functionalDB/README.md @@ -1,6 +1,6 @@ # FDB -FDB (TODO - fidn a better name) is an in-memory, no-sql functional database, written in Clojure. +FDB (TODO - find a better name) is an in-memory, no-sql functional database, written in Clojure. It is a modest attempt to provide part of the functionality that the Datomic database provides (the main omitted functionality is the durability part). From 298527da1814b1202bbb69913bbb9c10ff73af4d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 16 Feb 2014 12:46:12 +0200 Subject: [PATCH 023/102] adding what-if capability --- functionalDB/src/core/fdb.clj | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index a5433e5c1..e21ef67ec 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -9,18 +9,17 @@ (defn make-entity ([name] (make-entity :no-id-yet name)) ([id name] (Entity. id name {}))) +(defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:e_id attr-val) attr-val )) (defn make-attr[name val type] (Attr. name type (val-from-ref type val) -1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) -(defn nextId[db ent] ( - let [ topId (:topId db) - entId (:e_id ent) - [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] - [idToUse (keyword (str idToUse)) nextTop])) -(defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:e_id attr-val) attr-val )) +(defn nextId[db ent] (let [ topId (:topId db) + entId (:e_id ent) + [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] + [idToUse (keyword (str idToUse)) nextTop])) (defn update-creation-ts [ent tsVal] (let [ks (keys (:attrs ent)) @@ -37,8 +36,7 @@ attr-name (:name attr) back-reffing-set (get-in aevt [reffed-id attr-name] #{} ) new-back-reffing-set (operation back-reffing-set (:e_id ent)) - ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set) - )) + ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set))) (defn update-aevt[old-aevt ent operation] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) @@ -67,22 +65,27 @@ res (assoc db :timestamped (conj (:timestamped db) new-indices))] res)) -(defn transact-on-db [initial-db txs] +(defn transact-on-db [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] + (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [ initial-indices (:timestamped initial-db) new-indices (last (:timestamped transacted)) - transacted transacted res (assoc initial-db :timestamped (conj initial-indices new-indices) :curr-time (next-ts initial-db) :topId (:topId transacted)) ] - res)))) + res)))) -(defmacro transact [db & txs] +(defmacro transact_ [db op & txs] (when txs - (loop [[frst-tx# & rst-tx#] txs res# ['swap! db 'transact-on-db] cnt# []] - (if frst-tx# (recur rst-tx# res# (conj cnt# (vec frst-tx#))) - (list* (conj res# cnt#)))))) + (loop [[frst-tx# & rst-tx#] txs res# [op db 'transact-on-db] accum-txs# []] + (if frst-tx# (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) + (list* (conj res# accum-txs#)))))) + +(defn _what-if [ db f txs] (f db txs)) + +(defmacro what-if [db & txs] `(transact_ ~db _what-if ~@txs)) +(defmacro transact [db & txs] `(transact_ ~db swap! ~@txs)) (defn update-aevt-for-datom [aevt ent-id attr new-val] (if (not= :REF (:type attr )) From a86d3ee3af7de3e88d8f43d2c299ae92ccb59e1f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 17 Feb 2014 18:32:09 +0200 Subject: [PATCH 024/102] Improving what-if actions --- functionalDB/README.md | 2 ++ functionalDB/src/core/fdb.clj | 27 +++++++++++++-------------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/functionalDB/README.md b/functionalDB/README.md index e659681f8..d09b8c436 100644 --- a/functionalDB/README.md +++ b/functionalDB/README.md @@ -9,3 +9,5 @@ The core idea is to add the notion of time to the data, and provide on top of th The database itself is a collection of datums. A datum is built of an entity (think of a row in a table)that has its attributes (a column) and the value of that entity's attribute at a given time. Any update does not overwrite the prvious value, but addes another datum to the database. +The database supports both DB "modifying" (modifying as in creating a new DB value) transactions and what-if actions. + diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index e21ef67ec..3f4424647 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,23 +1,23 @@ (ns core.fdb) -(defrecord Entity [e_id name attrs]) +(defrecord Entity [id name attrs]) (defrecord Attr [name type value ts prev-ts]) (defn make-db[] ; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) - (atom {:timestamped [{:EAVT {} :AEVT {}}] - :topId 0 :curr-time 0})) + (atom {:timestamped [{:EAVT {} :AEVT {}}] :topId 0 :curr-time 0})) (defn make-entity ([name] (make-entity :no-id-yet name)) ([id name] (Entity. id name {}))) -(defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:e_id attr-val) attr-val )) -(defn make-attr[name val type] (Attr. name type (val-from-ref type val) -1 -1)) +(defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:id attr-val) attr-val )) + +(defn make-attr[name value type] (Attr. name type (val-from-ref type value) -1 -1)) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) (defn nextId[db ent] (let [ topId (:topId db) - entId (:e_id ent) + entId (:id ent) [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] [idToUse (keyword (str idToUse)) nextTop])) @@ -35,7 +35,7 @@ (let [reffed-id (:value attr) attr-name (:name attr) back-reffing-set (get-in aevt [reffed-id attr-name] #{} ) - new-back-reffing-set (operation back-reffing-set (:e_id ent)) + new-back-reffing-set (operation back-reffing-set (:id ent)) ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set))) (defn update-aevt[old-aevt ent operation] @@ -47,16 +47,15 @@ (defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) new-ts (next-ts db) indices (last (:timestamped db)) - fixed-ent (assoc ent :e_id ent-id-key) + fixed-ent (assoc ent :id ent-id-key) new-eavt (assoc (:EAVT indices) ent-id-key (update-creation-ts fixed-ent new-ts) ) new-aevt (update-aevt (:AEVT indices) fixed-ent conj) new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt ) - ](assoc db - :timestamped (conj (:timestamped db) new-indices) - :topId next-top))) + ](assoc db :timestamped (conj (:timestamped db) new-indices) + :topId next-top))) (defn remove-entity[db ent] - (let [ent-id (:e_id ent) + (let [ent-id (:id ent) indices (last (:timestamped db)) aevt (update-aevt (:AEVT indices) ent disj) new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity @@ -73,8 +72,8 @@ new-indices (last (:timestamped transacted)) res (assoc initial-db :timestamped (conj initial-indices new-indices) :curr-time (next-ts initial-db) - :topId (:topId transacted)) ] - res)))) + :topId (:topId transacted))] + res)))) (defmacro transact_ [db op & txs] (when txs From ec54c70e47e3b2229e68f6ccf3177b49d21eb42f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 17 Feb 2014 22:58:06 +0200 Subject: [PATCH 025/102] adding dbs and db-connection apis. For now creation and consumption --- functionalDB/src/core/manage.clj | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 functionalDB/src/core/manage.clj diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj new file mode 100644 index 000000000..d117adea8 --- /dev/null +++ b/functionalDB/src/core/manage.clj @@ -0,0 +1,15 @@ +(ns tmpTst (:use core.fdb)) + +(def all-dbs (atom {})) + +(defn _get-db [dbs db-name] + (if (db-name dbs ) dbs (assoc dbs db-name (make-db)))) + +(defn get-db-conn [db-name] + ((keyword db-name) (swap! all-dbs _get-db (keyword db-name)))) + +(defn db-from-conn [conn] @conn) + +(get-db-conn "aa") +(get-db-conn "ab") +@all-dbs From b7ae22d4be93f46becbc4f38731a108f42c4a50b Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 18 Feb 2014 00:20:35 +0200 Subject: [PATCH 026/102] adding the option to get entity by its id --- functionalDB/src/core/fdb.clj | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 3f4424647..a84bbe961 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -103,12 +103,15 @@ attr (get-in indices [:EAVT ent-id :attrs att-name] ) real-new-val (val-from-ref (:type attr) new-val) updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) - eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) - new-aevt (update-aevt-for-datom (:AEVT indices) ent-id attr real-new-val) - fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt) - new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) + eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) + new-aevt (update-aevt-for-datom (:AEVT indices) ent-id attr real-new-val) + fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt) + new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) ]new-db)) +(defn entity-at ([db ent-id ts] ((keyword ent-id) ((:timestamped db) ts))) + ([db ent-id] (entity-at db ent-id (:curr-time db))) ) + (defn attr-at "The attribute of an entity at a given time (defaults to recent time)" ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] @@ -118,7 +121,7 @@ ([db e-id attr-name] (:value (attr-at db e-id attr-name))) ([db e-id attr-name ts] (:value (attr-at db e-id attr-name ts)))) -(defn relates-to-as "returns a seq of all the entities that had an attribute named attr-name whose type is REF and the value was e-id, all this at a given time" +(defn relates-to-as "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is REF and the value was e-id), all this at a given time" ([db e-id attr-name] (relates-to-as db e-id attr-name (:curr-time db))) ([db e-id attr-name ts] (let [indices ((:timestamped db) ts) From cb4b3ea994ed65ca9e856ef8f0bd05c4289b7926 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 18 Feb 2014 12:27:57 +0200 Subject: [PATCH 027/102] using recrods instead of maps for a database and indices --- functionalDB/src/core/fdb.clj | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index a84bbe961..bd2ec04e1 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -2,9 +2,11 @@ (defrecord Entity [id name attrs]) (defrecord Attr [name type value ts prev-ts]) +(defrecord Database [timestamped top-id curr-time]) +(defrecord Indices [EAVT AEVT]) +(defn make-db[] (atom (Database. [(Indices. {} {})] 0 0) ;EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + )) ; -(defn make-db[] ; EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) - (atom {:timestamped [{:EAVT {} :AEVT {}}] :topId 0 :curr-time 0})) (defn make-entity ([name] (make-entity :no-id-yet name)) ([id name] (Entity. id name {}))) @@ -16,9 +18,9 @@ (defn next-ts [db] (inc (:curr-time db))) -(defn nextId[db ent] (let [ topId (:topId db) +(defn nextId[db ent] (let [ top-id (:top-id db) entId (:id ent) - [idToUse nextTop] (if (= entId :no-id-yet) [(inc topId) (inc topId)] [entId topId])] + [idToUse nextTop] (if (= entId :no-id-yet) [(inc top-id) (inc top-id)] [entId top-id])] [idToUse (keyword (str idToUse)) nextTop])) (defn update-creation-ts [ent tsVal] @@ -52,7 +54,7 @@ new-aevt (update-aevt (:AEVT indices) fixed-ent conj) new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt ) ](assoc db :timestamped (conj (:timestamped db) new-indices) - :topId next-top))) + :top-id next-top))) (defn remove-entity[db ent] (let [ent-id (:id ent) @@ -72,7 +74,7 @@ new-indices (last (:timestamped transacted)) res (assoc initial-db :timestamped (conj initial-indices new-indices) :curr-time (next-ts initial-db) - :topId (:topId transacted))] + :top-id (:top-id transacted))] res)))) (defmacro transact_ [db op & txs] From 4bd8f12f2356191f1fb3e35beafd22666429cd47 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 18 Feb 2014 12:31:33 +0200 Subject: [PATCH 028/102] cleanup --- functionalDB/src/core/manage.clj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index d117adea8..d921b91d6 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -9,7 +9,3 @@ ((keyword db-name) (swap! all-dbs _get-db (keyword db-name)))) (defn db-from-conn [conn] @conn) - -(get-db-conn "aa") -(get-db-conn "ab") -@all-dbs From 720ff0b88c368bd785a5efa92c04c541d228a5ef Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 18 Feb 2014 23:02:44 +0200 Subject: [PATCH 029/102] adding basic AVET indexing --- functionalDB/src/core/fdb.clj | 44 +++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index bd2ec04e1..15f0208a0 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -3,7 +3,7 @@ (defrecord Entity [id name attrs]) (defrecord Attr [name type value ts prev-ts]) (defrecord Database [timestamped top-id curr-time]) -(defrecord Indices [EAVT AEVT]) +(defrecord Indices [EAVT AEVT AVET]) (defn make-db[] (atom (Database. [(Indices. {} {})] 0 0) ;EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) )) ; @@ -13,7 +13,10 @@ (defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:id attr-val) attr-val )) -(defn make-attr[name value type] (Attr. name type (val-from-ref type value) -1 -1)) +(defn make-attr + ([name value type] (make-attr name value type false)) + ([name value type indexed] (with-meta (Attr. name type (val-from-ref type value) -1 -1) {:indexed indexed} ))) + (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) (defn next-ts [db] (inc (:curr-time db))) @@ -40,11 +43,25 @@ new-back-reffing-set (operation back-reffing-set (:id ent)) ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set))) +(defn update-attr-in-avet[ent operation avet attr] + (let [attr-name (:name attr) + attr-value (:value attr) + curr-entities-set (get-in avet [attr-name attr-value] {}) + updated-entities-set (operation curr-entities-set (:id ent)) + ] (assoc-in avet [attr-name attr-value] updated-entities-set)) + ) + (defn update-aevt[old-aevt ent operation] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) add-ref (partial add-ref-to-aevt ent operation)] (reduce add-ref old-aevt reffingAttrs))) +;avet : attr-name -> attr-value -> #{ids of ents} +(update-avet [old-avet ent operation] + (let [indexed-attrs (filter #(:indexed (meta %)) (vals (:attrs ent))) + update-attr-in-avet-fn (partial update-attr-in-avet ent operation)] + (reduce add-attr-to-avet-fn old-avet indexed-attrs))) + ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) new-ts (next-ts db) @@ -52,7 +69,8 @@ fixed-ent (assoc ent :id ent-id-key) new-eavt (assoc (:EAVT indices) ent-id-key (update-creation-ts fixed-ent new-ts) ) new-aevt (update-aevt (:AEVT indices) fixed-ent conj) - new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt ) + new-avet (update-avet (:AVET indices) fixed-ent conj) + new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt :AVET new-avet ) ](assoc db :timestamped (conj (:timestamped db) new-indices) :top-id next-top))) @@ -60,9 +78,10 @@ (let [ent-id (:id ent) indices (last (:timestamped db)) aevt (update-aevt (:AEVT indices) ent disj) + avet (update-avet (:AVET indices) ent disj) new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity new-aevt (dissoc aevt ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt) + new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt :AVET avet) res (assoc db :timestamped (conj (:timestamped db) new-indices))] res)) @@ -99,6 +118,17 @@ updated-aevt (assoc-in cleaned-aevt [new-val attr-name] (conj to-be-updated-ref ent-id) ) ] updated-aevt))) +(defn update-avet-for-datom [avet ent-id attr new-val] + (if (:indexed (meta attr)) + avet + (let [old-attr-val (:value attr) + attr-name (:name attr) + old-entities-set (get-in avet [attr-name attr-val]) + cleaned-avet (assoc-in avet [attr-name attr-val] (disj old-entities-set ent-id)) + to-be-updated-entities-set (get-in avet [attr-name new-val] #{}) + updated-avet (assoc-in cleaned-avet [attr-name new-val] (conj to-be-updated-entities-set ent-id)) + ] updated-avet ))) + (defn update-datom [db ent-id att-name new-val] (let [ new-ts (next-ts db) indices (last (:timestamped db)) @@ -107,7 +137,8 @@ updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) new-aevt (update-aevt-for-datom (:AEVT indices) ent-id attr real-new-val) - fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt) + new-avet (update-avet-for-datom (:AVET indices) ent-id attr real-new-val) + fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt :AVET new-avet) new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) ]new-db)) @@ -127,8 +158,7 @@ ([db e-id attr-name] (relates-to-as db e-id attr-name (:curr-time db))) ([db e-id attr-name ts] (let [indices ((:timestamped db) ts) - reffing-ids (get-in indices [:AEVT e-id attr-name]) - ] + reffing-ids (get-in indices [:AEVT e-id attr-name])] (map #(get-in indices [:EAVT %]) reffing-ids )))) (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] From f7f8762c266b202746694e37ec6e74778f4b78fb Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 19 Feb 2014 12:06:33 +0200 Subject: [PATCH 030/102] fixing incosistencies bug in AVET --- functionalDB/src/core/fdb.clj | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 15f0208a0..b7e8af3f5 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -4,18 +4,18 @@ (defrecord Attr [name type value ts prev-ts]) (defrecord Database [timestamped top-id curr-time]) (defrecord Indices [EAVT AEVT AVET]) -(defn make-db[] (atom (Database. [(Indices. {} {})] 0 0) ;EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) +(defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0) ;EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) )) ; (defn make-entity ([name] (make-entity :no-id-yet name)) - ([id name] (Entity. id name {}))) + ([name id] (Entity. id name {}))) (defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:id attr-val) attr-val )) (defn make-attr ([name value type] (make-attr name value type false)) - ([name value type indexed] (with-meta (Attr. name type (val-from-ref type value) -1 -1) {:indexed indexed} ))) + ([name value type indexed] (with-meta (Attr. name type value -1 -1) {:indexed indexed} ))) (defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) @@ -23,8 +23,8 @@ (defn nextId[db ent] (let [ top-id (:top-id db) entId (:id ent) - [idToUse nextTop] (if (= entId :no-id-yet) [(inc top-id) (inc top-id)] [entId top-id])] - [idToUse (keyword (str idToUse)) nextTop])) + [idToUse nextTop] (if (= entId :no-id-yet) [(keyword (str (inc top-id))) (inc top-id)] [entId top-id])] + [idToUse nextTop])) (defn update-creation-ts [ent tsVal] (let [ks (keys (:attrs ent)) @@ -46,7 +46,7 @@ (defn update-attr-in-avet[ent operation avet attr] (let [attr-name (:name attr) attr-value (:value attr) - curr-entities-set (get-in avet [attr-name attr-value] {}) + curr-entities-set (get-in avet [attr-name attr-value] #{}) updated-entities-set (operation curr-entities-set (:id ent)) ] (assoc-in avet [attr-name attr-value] updated-entities-set)) ) @@ -57,17 +57,17 @@ (reduce add-ref old-aevt reffingAttrs))) ;avet : attr-name -> attr-value -> #{ids of ents} -(update-avet [old-avet ent operation] +(defn update-avet [old-avet ent operation] (let [indexed-attrs (filter #(:indexed (meta %)) (vals (:attrs ent))) update-attr-in-avet-fn (partial update-attr-in-avet ent operation)] - (reduce add-attr-to-avet-fn old-avet indexed-attrs))) + (reduce update-attr-in-avet-fn old-avet indexed-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity[db ent] (let [[ent-id ent-id-key next-top] (nextId db ent) +(defn add-entity[db ent] (let [[ent-id next-top] (nextId db ent) new-ts (next-ts db) indices (last (:timestamped db)) - fixed-ent (assoc ent :id ent-id-key) - new-eavt (assoc (:EAVT indices) ent-id-key (update-creation-ts fixed-ent new-ts) ) + fixed-ent (assoc ent :id ent-id) + new-eavt (assoc (:EAVT indices) ent-id (update-creation-ts fixed-ent new-ts) ) new-aevt (update-aevt (:AEVT indices) fixed-ent conj) new-avet (update-avet (:AVET indices) fixed-ent conj) new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt :AVET new-avet ) @@ -123,8 +123,8 @@ avet (let [old-attr-val (:value attr) attr-name (:name attr) - old-entities-set (get-in avet [attr-name attr-val]) - cleaned-avet (assoc-in avet [attr-name attr-val] (disj old-entities-set ent-id)) + old-entities-set (get-in avet [attr-name old-attr-val]) + cleaned-avet (assoc-in avet [attr-name old-attr-val] (disj old-entities-set ent-id)) to-be-updated-entities-set (get-in avet [attr-name new-val] #{}) updated-avet (assoc-in cleaned-avet [attr-name new-val] (conj to-be-updated-entities-set ent-id)) ] updated-avet ))) From 39286522aab3924d558540d6a05a2756062ed254 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 19 Feb 2014 14:28:40 +0200 Subject: [PATCH 031/102] fixing name of aevt to vaet --- functionalDB/src/core/fdb.clj | 48 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index b7e8af3f5..79c217e56 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -3,8 +3,8 @@ (defrecord Entity [id name attrs]) (defrecord Attr [name type value ts prev-ts]) (defrecord Database [timestamped top-id curr-time]) -(defrecord Indices [EAVT AEVT AVET]) -(defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0) ;EAVT: all the entity info, AEVT for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) +(defrecord Indices [EAVT VAET AVET]) +(defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) )) ; @@ -33,15 +33,15 @@ updatedAttrs (zipmap ks updatedAttrsVals) ](assoc ent :attrs updatedAttrs))) -; AEVT -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} +; vaet -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} ; this basically provides the info - for each entity that is REFFed by others, who are the others who are REFing it, separated ; by the names of the attribute used for reffing -(defn add-ref-to-aevt[ent operation aevt attr] +(defn add-ref-to-vaet[ent operation vaet attr] (let [reffed-id (:value attr) attr-name (:name attr) - back-reffing-set (get-in aevt [reffed-id attr-name] #{} ) + back-reffing-set (get-in vaet [reffed-id attr-name] #{} ) new-back-reffing-set (operation back-reffing-set (:id ent)) - ] (assoc-in aevt [reffed-id attr-name] new-back-reffing-set))) + ] (assoc-in vaet [reffed-id attr-name] new-back-reffing-set))) (defn update-attr-in-avet[ent operation avet attr] (let [attr-name (:name attr) @@ -51,10 +51,10 @@ ] (assoc-in avet [attr-name attr-value] updated-entities-set)) ) -(defn update-aevt[old-aevt ent operation] +(defn update-vaet[old-vaet ent operation] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) - add-ref (partial add-ref-to-aevt ent operation)] - (reduce add-ref old-aevt reffingAttrs))) + add-ref (partial add-ref-to-vaet ent operation)] + (reduce add-ref old-vaet reffingAttrs))) ;avet : attr-name -> attr-value -> #{ids of ents} (defn update-avet [old-avet ent operation] @@ -68,20 +68,20 @@ indices (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) new-eavt (assoc (:EAVT indices) ent-id (update-creation-ts fixed-ent new-ts) ) - new-aevt (update-aevt (:AEVT indices) fixed-ent conj) + new-vaet (update-vaet (:VAET indices) fixed-ent conj) new-avet (update-avet (:AVET indices) fixed-ent conj) - new-indices (assoc indices :AEVT new-aevt :EAVT new-eavt :AVET new-avet ) + new-indices (assoc indices :VAET new-vaet :EAVT new-eavt :AVET new-avet ) ](assoc db :timestamped (conj (:timestamped db) new-indices) :top-id next-top))) (defn remove-entity[db ent] (let [ent-id (:id ent) indices (last (:timestamped db)) - aevt (update-aevt (:AEVT indices) ent disj) + vaet (update-vaet (:VAET indices) ent disj) avet (update-avet (:AVET indices) ent disj) new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity - new-aevt (dissoc aevt ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :AEVT new-aevt :AVET avet) + new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity + new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet) res (assoc db :timestamped (conj (:timestamped db) new-indices))] res)) @@ -107,16 +107,16 @@ (defmacro what-if [db & txs] `(transact_ ~db _what-if ~@txs)) (defmacro transact [db & txs] `(transact_ ~db swap! ~@txs)) -(defn update-aevt-for-datom [aevt ent-id attr new-val] +(defn update-vaet-for-datom [vaet ent-id attr new-val] (if (not= :REF (:type attr )) - aevt + vaet (let [ old-ref-id (:value attr) attr-name (:name attr) - old-reffed (get-in aevt [old-ref-id attr-name]) - cleaned-aevt (assoc-in aevt [old-ref-id attr-name] (disj old-reffed ent-id)) - to-be-updated-ref (get-in aevt [new-val attr-name] #{}) - updated-aevt (assoc-in cleaned-aevt [new-val attr-name] (conj to-be-updated-ref ent-id) ) - ] updated-aevt))) + old-reffed (get-in vaet [old-ref-id attr-name]) + cleaned-vaet (assoc-in vaet [old-ref-id attr-name] (disj old-reffed ent-id)) + to-be-updated-ref (get-in vaet [new-val attr-name] #{}) + updated-vaet (assoc-in cleaned-vaet [new-val attr-name] (conj to-be-updated-ref ent-id) ) + ] updated-vaet))) (defn update-avet-for-datom [avet ent-id attr new-val] (if (:indexed (meta attr)) @@ -136,9 +136,9 @@ real-new-val (val-from-ref (:type attr) new-val) updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) - new-aevt (update-aevt-for-datom (:AEVT indices) ent-id attr real-new-val) + new-vaet (update-vaet-for-datom (:VAET indices) ent-id attr real-new-val) new-avet (update-avet-for-datom (:AVET indices) ent-id attr real-new-val) - fully-updated-indices (assoc eavt-updated-indices :AEVT new-aevt :AVET new-avet) + fully-updated-indices (assoc eavt-updated-indices :VAET new-vaet :AVET new-avet) new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) ]new-db)) @@ -158,7 +158,7 @@ ([db e-id attr-name] (relates-to-as db e-id attr-name (:curr-time db))) ([db e-id attr-name ts] (let [indices ((:timestamped db) ts) - reffing-ids (get-in indices [:AEVT e-id attr-name])] + reffing-ids (get-in indices [:VAET e-id attr-name])] (map #(get-in indices [:EAVT %]) reffing-ids )))) (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] From e7fb342899d88d477c6d769b69f21a909ae7c465 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 21 Feb 2014 08:40:59 +0200 Subject: [PATCH 032/102] entities shouldn't have a name, only id adding the before operation on a db --- functionalDB/src/core/fdb.clj | 37 ++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 79c217e56..a0ed40b8b 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,17 +1,15 @@ (ns core.fdb) -(defrecord Entity [id name attrs]) -(defrecord Attr [name type value ts prev-ts]) (defrecord Database [timestamped top-id curr-time]) (defrecord Indices [EAVT VAET AVET]) -(defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) - )) ; +(defrecord Entity [id attrs]) +(defrecord Attr [name type value ts prev-ts]) -(defn make-entity ([name] (make-entity :no-id-yet name)) - ([name id] (Entity. id name {}))) +(defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) -(defn val-from-ref[attr-type attr-val](if (= :REF attr-type) (:id attr-val) attr-val )) +(defn make-entity ([] (make-entity :db/no-id-yet )) + ([name id] (Entity. id {}))) (defn make-attr ([name value type] (make-attr name value type false)) @@ -23,7 +21,7 @@ (defn nextId[db ent] (let [ top-id (:top-id db) entId (:id ent) - [idToUse nextTop] (if (= entId :no-id-yet) [(keyword (str (inc top-id))) (inc top-id)] [entId top-id])] + [idToUse nextTop] (if (= entId :db/no-id-yet) [(keyword (str (inc top-id))) (inc top-id)] [entId top-id])] [idToUse nextTop])) (defn update-creation-ts [ent tsVal] @@ -34,7 +32,7 @@ ](assoc ent :attrs updatedAttrs))) ; vaet -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} -; this basically provides the info - for each entity that is REFFed by others, who are the others who are REFing it, separated +; this basically provides this info - for each entity that is REFFed by others, who are the others who are REFing it, separated ; by the names of the attribute used for reffing (defn add-ref-to-vaet[ent operation vaet attr] (let [reffed-id (:value attr) @@ -48,8 +46,7 @@ attr-value (:value attr) curr-entities-set (get-in avet [attr-name attr-value] #{}) updated-entities-set (operation curr-entities-set (:id ent)) - ] (assoc-in avet [attr-name attr-value] updated-entities-set)) - ) + ] (assoc-in avet [attr-name attr-value] updated-entities-set))) (defn update-vaet[old-vaet ent operation] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) @@ -133,7 +130,7 @@ (let [ new-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name] ) - real-new-val (val-from-ref (:type attr) new-val) + real-new-val (if (= :REF (:type attr)) (:id new-val) new-val) updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) new-vaet (update-vaet-for-datom (:VAET indices) ent-id attr real-new-val) @@ -151,14 +148,14 @@ (let [indices ((:timestamped db) ts)] (get-in indices [:EAVT ent-id :attrs attr-name])))) (defn value-of-at "value of a datom at a given time, if no time is provided, we default to the most recent value" - ([db e-id attr-name] (:value (attr-at db e-id attr-name))) - ([db e-id attr-name ts] (:value (attr-at db e-id attr-name ts)))) + ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) + ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) -(defn relates-to-as "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is REF and the value was e-id), all this at a given time" - ([db e-id attr-name] (relates-to-as db e-id attr-name (:curr-time db))) - ([db e-id attr-name ts] +(defn relates-to-as "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is REF and the value was ent-id), all this at a given time" + ([db ent-id attr-name] (relates-to-as db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name ts] (let [indices ((:timestamped db) ts) - reffing-ids (get-in indices [:VAET e-id attr-name])] + reffing-ids (get-in indices [:VAET ent-id attr-name])] (map #(get-in indices [:EAVT %]) reffing-ids )))) (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] @@ -166,3 +163,7 @@ (if (= -1 ts) (reverse res) (let [attr (attr-at db ent-id attr-name ts)] (recur (conj res {ts (:value attr)}) (:prev-ts attr)))))) + +(defn db-before [db ts] + (let [indices-before (subvec (:timestamped db) 0 ts )] + (assoc db :timestamped indices-before :curr-time ts))) From cb7b7d6388026c8aa925df3b90d675f81191537b Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 22 Feb 2014 15:30:55 +0200 Subject: [PATCH 033/102] basic support for multiple cardinality --- functionalDB/src/core/fdb.clj | 128 ++++++++++++++++++++++++---------- 1 file changed, 93 insertions(+), 35 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index a0ed40b8b..8a8b05684 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -2,26 +2,53 @@ (defrecord Database [timestamped top-id curr-time]) (defrecord Indices [EAVT VAET AVET]) - (defrecord Entity [id attrs]) (defrecord Attr [name type value ts prev-ts]) (defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) (defn make-entity ([] (make-entity :db/no-id-yet )) - ([name id] (Entity. id {}))) + ([id] (Entity. id {}))) (defn make-attr - ([name value type] (make-attr name value type false)) - ([name value type indexed] (with-meta (Attr. name type value -1 -1) {:indexed indexed} ))) - -(defn add-attr[ ent attr] (assoc-in ent [:attrs (keyword (:name attr))] attr)) - -(defn next-ts [db] (inc (:curr-time db))) - -(defn nextId[db ent] (let [ top-id (:top-id db) - entId (:id ent) - [idToUse nextTop] (if (= entId :db/no-id-yet) [(keyword (str (inc top-id))) (inc top-id)] [entId top-id])] + ([name value type ] (make-attr name value type :db/single)) + ([name value type cardinality] (make-attr name value type cardinality false)) + ([name value type cardinality indexed] (with-meta (Attr. name type value -1 -1) {:indexed indexed :cardinality cardinality} ))) + +(defn update-attr-value [attr value operation] + (cond + (= :db/single (:cardinality (meta attr))) (assoc attr :value #{value}) + ; now = :db/mutiple (:cardinality (meta attr))) + + (= :db/reset-to operation) (assoc attr :value #{value}) + (= :db/add operation) (assoc attr :value (conj (:value attr) value )) + (= :db/remove operation) (assoc attr :value (disj (:value attr) value )) + )) + +;; (if (= :db/single (:cardinality (meta attr))) +;; (assoc attr :value #{value}) +;; (assoc attr :value (conj (:value attr) value)))) + +(defn conj-to-attr [old new operation] + (let [new-val (:value new)] + (if old + (update-attr-value old (:value new) operation) + (update-attr-value new (:value new) :db/reset-to )))) + +(defn add-attr[ ent attr] + (let [attr-id (keyword (:name attr)) + existing-attr (get-in ent [:attrs attr-id]) + updated-attr (conj-to-attr existing-attr attr :db/add)] + (assoc-in ent [:attrs attr-id ] updated-attr))) + +(defn next-ts [db] + (inc (:curr-time db))) + +(defn nextId[db ent] + (let [top-id (:top-id db) + entId (:id ent) + inceased-id (inc top-id) + [idToUse nextTop] (if (= entId :db/no-id-yet) [(keyword (str inceased-id)) inceased-id] [entId top-id])] [idToUse nextTop])) (defn update-creation-ts [ent tsVal] @@ -35,7 +62,7 @@ ; this basically provides this info - for each entity that is REFFed by others, who are the others who are REFing it, separated ; by the names of the attribute used for reffing (defn add-ref-to-vaet[ent operation vaet attr] - (let [reffed-id (:value attr) + (let [reffed-id (first (:value attr)) attr-name (:name attr) back-reffing-set (get-in vaet [reffed-id attr-name] #{} ) new-back-reffing-set (operation back-reffing-set (:id ent)) @@ -43,7 +70,7 @@ (defn update-attr-in-avet[ent operation avet attr] (let [attr-name (:name attr) - attr-value (:value attr) + attr-value (first (:value attr)) ; a set curr-entities-set (get-in avet [attr-name attr-value] #{}) updated-entities-set (operation curr-entities-set (:id ent)) ] (assoc-in avet [attr-name attr-value] updated-entities-set))) @@ -60,7 +87,8 @@ (reduce update-attr-in-avet-fn old-avet indexed-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity[db ent] (let [[ent-id next-top] (nextId db ent) +(defn add-entity[db ent] + (let [[ent-id next-top] (nextId db ent) new-ts (next-ts db) indices (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) @@ -102,42 +130,72 @@ (defn _what-if [ db f txs] (f db txs)) (defmacro what-if [db & txs] `(transact_ ~db _what-if ~@txs)) + (defmacro transact [db & txs] `(transact_ ~db swap! ~@txs)) -(defn update-vaet-for-datom [vaet ent-id attr new-val] +(defn remove-entry-from-index [index path val-to-remove operation] + (if (= operation :db/add) + index + (let [old-entries-set (get-in index path )] + (assoc-in index path (disj old-entries-set val-to-remove))))) + +(defn add-entry-to-index [index path val-to-add operation] + (if (= operation :db/remove) + index + (let [to-be-updated-set (get-in index path #{})] + (assoc-in index path (conj to-be-updated-set val-to-add))))) + +(defn update-vaet-for-datom [vaet ent-id attr new-ref-id operation] (if (not= :REF (:type attr )) vaet - (let [ old-ref-id (:value attr) - attr-name (:name attr) - old-reffed (get-in vaet [old-ref-id attr-name]) - cleaned-vaet (assoc-in vaet [old-ref-id attr-name] (disj old-reffed ent-id)) - to-be-updated-ref (get-in vaet [new-val attr-name] #{}) - updated-vaet (assoc-in cleaned-vaet [new-val attr-name] (conj to-be-updated-ref ent-id) ) + (let [ + old-ref-id (first (:value attr)) + attr-name (:name attr) + ;; removing old value + cleaned-vaet (remove-entry-from-index vaet [old-ref-id attr-name] ent-id) + +;; old-reffed (get-in vaet [old-ref-id attr-name]) +;; cleaned-vaet (assoc-in vaet [old-ref-id attr-name] (disj old-reffed ent-id)) + ;; adding new value + updated-vaet (add-entry-to-index cleaned-vaet [new-ref-id attr-name] ent-id operation) +;; to-be-updated-ref (get-in cleaned-vaet [new-val attr-name] #{}) +;; updated-vaet (assoc-in cleaned-vaet [new-val attr-name] (conj to-be-updated-ref ent-id)) ] updated-vaet))) -(defn update-avet-for-datom [avet ent-id attr new-val] +(defn update-avet-for-datom [avet ent-id attr new-attr-val operation] (if (:indexed (meta attr)) avet - (let [old-attr-val (:value attr) + (let [old-attr-val (first (:value attr)) attr-name (:name attr) - old-entities-set (get-in avet [attr-name old-attr-val]) - cleaned-avet (assoc-in avet [attr-name old-attr-val] (disj old-entities-set ent-id)) - to-be-updated-entities-set (get-in avet [attr-name new-val] #{}) - updated-avet (assoc-in cleaned-avet [attr-name new-val] (conj to-be-updated-entities-set ent-id)) + ;; removing old value + cleaned-avet (remove-entry-from-index avet [attr-name old-attr-val] ent-id operation) + +;; old-entities-set (get-in avet [attr-name old-attr-val]) +;; cleaned-avet (assoc-in avet [attr-name old-attr-val] (disj old-entities-set ent-id)) + ;; adding new value + updated-avet (add-entry-to-index cleaned-avet [attr-name new-attr-val] ent-id operation) +;; to-be-updated-entities-set (get-in cleaned-avet [attr-name new-val] #{}) +;; updated-avet (assoc-in cleaned-avet [attr-name new-val] (conj to-be-updated-entities-set ent-id)) ] updated-avet ))) -(defn update-datom [db ent-id att-name new-val] +(defn update-eavt-for-datom [eavt ent-id new-attr] + (assoc-in eavt [ent-id (:name new-attr)] new-attr)) + +(defn update-datom + ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) + ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/mutiple) (let [ new-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name] ) real-new-val (if (= :REF (:type attr)) (:id new-val) new-val) - updated-attr(assoc attr :value real-new-val :ts new-ts :prev-ts ( :ts attr)) - eavt-updated-indices (assoc-in indices [:EAVT ent-id :attrs att-name] updated-attr ) - new-vaet (update-vaet-for-datom (:VAET indices) ent-id attr real-new-val) - new-avet (update-avet-for-datom (:AVET indices) ent-id attr real-new-val) - fully-updated-indices (assoc eavt-updated-indices :VAET new-vaet :AVET new-avet) + updated-timed-attr (assoc attr :ts new-ts :prev-ts ( :ts attr)) + updated-attr (conj-to-attr updated-timed-attr {:value real-new-val} operation) + new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) + new-vaet (update-vaet-for-datom (:VAET indices) ent-id attr real-new-val operation) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-avet (update-avet-for-datom (:AVET indices) ent-id attr real-new-val operation) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + fully-updated-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet) new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) - ]new-db)) + ]new-db))) (defn entity-at ([db ent-id ts] ((keyword ent-id) ((:timestamped db) ts))) ([db ent-id] (entity-at db ent-id (:curr-time db))) ) From eb9e3990b0e7892702961d06435c3540afb63411 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 23 Feb 2014 10:21:59 +0200 Subject: [PATCH 034/102] Proper support for :db/mutiple - fixing issues with updating the indices --- functionalDB/src/core/fdb.clj | 70 +++++++++++++++++++---------------- 1 file changed, 39 insertions(+), 31 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 8a8b05684..453a7642f 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -19,16 +19,11 @@ (cond (= :db/single (:cardinality (meta attr))) (assoc attr :value #{value}) ; now = :db/mutiple (:cardinality (meta attr))) - (= :db/reset-to operation) (assoc attr :value #{value}) (= :db/add operation) (assoc attr :value (conj (:value attr) value )) (= :db/remove operation) (assoc attr :value (disj (:value attr) value )) )) -;; (if (= :db/single (:cardinality (meta attr))) -;; (assoc attr :value #{value}) -;; (assoc attr :value (conj (:value attr) value)))) - (defn conj-to-attr [old new operation] (let [new-val (:value new)] (if old @@ -129,15 +124,25 @@ (defn _what-if [ db f txs] (f db txs)) -(defmacro what-if [db & txs] `(transact_ ~db _what-if ~@txs)) +(defmacro what-if [db & txs] + `(transact_ ~db _what-if ~@txs)) + +(defmacro transact [db & txs] + `(transact_ ~db swap! ~@txs)) -(defmacro transact [db & txs] `(transact_ ~db swap! ~@txs)) +;;;;;;;;;;;;;;;;;;;;;;; datom updates -(defn remove-entry-from-index [index path val-to-remove operation] +(defn remove-entry-from-index [val-to-remove index path ] + (let [ old-entries-set (get-in index path )] + (assoc-in index path (disj old-entries-set val-to-remove)) + )) + +(defn remove-entries-from-index [index paths val-to-remove operation] (if (= operation :db/add) index - (let [old-entries-set (get-in index path )] - (assoc-in index path (disj old-entries-set val-to-remove))))) + (let [ remover (partial remove-entry-from-index val-to-remove) ] + (reduce remover index (operation paths)) + ))) (defn add-entry-to-index [index path val-to-add operation] (if (= operation :db/remove) @@ -145,41 +150,39 @@ (let [to-be-updated-set (get-in index path #{})] (assoc-in index path (conj to-be-updated-set val-to-add))))) -(defn update-vaet-for-datom [vaet ent-id attr new-ref-id operation] +(defn update-vaet-for-datom [vaet ent-id attr target-ref-id operation] (if (not= :REF (:type attr )) vaet (let [ - old-ref-id (first (:value attr)) + old-ref-ids (:value attr) attr-name (:name attr) - ;; removing old value - cleaned-vaet (remove-entry-from-index vaet [old-ref-id attr-name] ent-id) - -;; old-reffed (get-in vaet [old-ref-id attr-name]) -;; cleaned-vaet (assoc-in vaet [old-ref-id attr-name] (disj old-reffed ent-id)) + paths-in-case-of-replace (map (fn[i][i attr-name]) old-ref-ids) + paths-in-case-of-remove [[target-ref-id attr-name]] + paths {:db/remove paths-in-case-of-remove :db/reset-to paths-in-case-of-replace} + ;; removing old values + cleaned-vaet (remove-entries-from-index vaet paths ent-id operation) ;; adding new value - updated-vaet (add-entry-to-index cleaned-vaet [new-ref-id attr-name] ent-id operation) -;; to-be-updated-ref (get-in cleaned-vaet [new-val attr-name] #{}) -;; updated-vaet (assoc-in cleaned-vaet [new-val attr-name] (conj to-be-updated-ref ent-id)) + updated-vaet (add-entry-to-index cleaned-vaet [target-ref-id attr-name] ent-id operation) ] updated-vaet))) -(defn update-avet-for-datom [avet ent-id attr new-attr-val operation] - (if (:indexed (meta attr)) +(defn update-avet-for-datom [avet ent-id attr target-attr-val operation] + (if-not (:indexed (meta attr)) avet - (let [old-attr-val (first (:value attr)) + (let [old-attr-vals (:value attr) attr-name (:name attr) + paths-in-case-of-replace (map (fn[i][attr-name i]) old-attr-vals) + paths-in-case-of-remove [[attr-name target-attr-val]] + paths {:db/remove paths-in-case-of-remove :db/reset-to paths-in-case-of-replace} ;; removing old value - cleaned-avet (remove-entry-from-index avet [attr-name old-attr-val] ent-id operation) - -;; old-entities-set (get-in avet [attr-name old-attr-val]) -;; cleaned-avet (assoc-in avet [attr-name old-attr-val] (disj old-entities-set ent-id)) + cleaned-avet (remove-entries-from-index avet paths ent-id operation) ;; adding new value - updated-avet (add-entry-to-index cleaned-avet [attr-name new-attr-val] ent-id operation) -;; to-be-updated-entities-set (get-in cleaned-avet [attr-name new-val] #{}) -;; updated-avet (assoc-in cleaned-avet [attr-name new-val] (conj to-be-updated-entities-set ent-id)) + updated-avet (add-entry-to-index cleaned-avet [attr-name target-attr-val] ent-id operation) ] updated-avet ))) (defn update-eavt-for-datom [eavt ent-id new-attr] - (assoc-in eavt [ent-id (:name new-attr)] new-attr)) + ( + + assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) (defn update-datom ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) @@ -197,6 +200,8 @@ new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) ]new-db))) +;;;;;;;;;;;;;;;;;;;;;;; queries + (defn entity-at ([db ent-id ts] ((keyword ent-id) ((:timestamped db) ts))) ([db ent-id] (entity-at db ent-id (:curr-time db))) ) @@ -225,3 +230,6 @@ (defn db-before [db ts] (let [indices-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped indices-before :curr-time ts))) + +(defn ind-at[db ts kind] (kind ((:timestamped db) ts))) + From 2dc8a5f0b7dbfaafea6aac889d47982bebe08682 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 24 Feb 2014 08:43:00 +0200 Subject: [PATCH 035/102] adding doc strings --- functionalDB/src/core/fdb.clj | 44 +++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 453a7642f..d4dc0f6f3 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -5,9 +5,11 @@ (defrecord Entity [id attrs]) (defrecord Attr [name type value ts prev-ts]) -(defn make-db[] (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) +(defn make-db "Create an empty database" [] + (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) -(defn make-entity ([] (make-entity :db/no-id-yet )) +(defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" + ([] (make-entity :db/no-id-yet )) ([id] (Entity. id {}))) (defn make-attr @@ -15,7 +17,8 @@ ([name value type cardinality] (make-attr name value type cardinality false)) ([name value type cardinality indexed] (with-meta (Attr. name type value -1 -1) {:indexed indexed :cardinality cardinality} ))) -(defn update-attr-value [attr value operation] +(defn update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" + [attr value operation] (cond (= :db/single (:cardinality (meta attr))) (assoc attr :value #{value}) ; now = :db/mutiple (:cardinality (meta attr))) @@ -24,46 +27,54 @@ (= :db/remove operation) (assoc attr :value (disj (:value attr) value )) )) -(defn conj-to-attr [old new operation] +(defn conj-to-attr "conjoins a value to an attribute, this function can be used either when initializing a new attribute or updating an existing one" + [old new operation] (let [new-val (:value new)] (if old (update-attr-value old (:value new) operation) (update-attr-value new (:value new) :db/reset-to )))) -(defn add-attr[ ent attr] +(defn add-attr "adds an attribute to an entity" + [ ent attr] (let [attr-id (keyword (:name attr)) existing-attr (get-in ent [:attrs attr-id]) updated-attr (conj-to-attr existing-attr attr :db/add)] (assoc-in ent [:attrs attr-id ] updated-attr))) -(defn next-ts [db] +(defn next-ts [db] "returns the next timestamp of the given database" (inc (:curr-time db))) -(defn nextId[db ent] +(defn nextId [db ent] "returns a pair composed of the id to use for the given entity and the next free running id in the database" (let [top-id (:top-id db) entId (:id ent) inceased-id (inc top-id) [idToUse nextTop] (if (= entId :db/no-id-yet) [(keyword (str inceased-id)) inceased-id] [entId top-id])] [idToUse nextTop])) -(defn update-creation-ts [ent tsVal] +(defn update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" + [ent tsVal] (let [ks (keys (:attrs ent)) vls (vals (:attrs ent)) updatedAttrsVals (map #(assoc % :ts tsVal) vls) updatedAttrs (zipmap ks updatedAttrsVals) ](assoc ent :attrs updatedAttrs))) -; vaet -> {REFed-ent-id -> {attrName -> [REFing-elems-ids]}} -; this basically provides this info - for each entity that is REFFed by others, who are the others who are REFing it, separated -; by the names of the attribute used for reffing -(defn add-ref-to-vaet[ent operation vaet attr] +; VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} +; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) +; can be used to know who REFs a specific entity +(defn update-ref-in-vaet "adding an entry to th VAET index" + [ent operation vaet attr] (let [reffed-id (first (:value attr)) attr-name (:name attr) back-reffing-set (get-in vaet [reffed-id attr-name] #{} ) new-back-reffing-set (operation back-reffing-set (:id ent)) ] (assoc-in vaet [reffed-id attr-name] new-back-reffing-set))) -(defn update-attr-in-avet[ent operation avet attr] +; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} +; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) +; can be used to know who are the entities that have a specific attribute with a specific value +(defn update-attr-in-avet "" + [ent operation avet attr] (let [attr-name (:name attr) attr-value (first (:value attr)) ; a set curr-entities-set (get-in avet [attr-name attr-value] #{}) @@ -72,7 +83,7 @@ (defn update-vaet[old-vaet ent operation] (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) - add-ref (partial add-ref-to-vaet ent operation)] + add-ref (partial update-ref-in-vaet ent operation)] (reduce add-ref old-vaet reffingAttrs))) ;avet : attr-name -> attr-value -> #{ids of ents} @@ -180,9 +191,7 @@ ] updated-avet ))) (defn update-eavt-for-datom [eavt ent-id new-attr] - ( - - assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) + (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) (defn update-datom ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) @@ -232,4 +241,3 @@ (assoc db :timestamped indices-before :curr-time ts))) (defn ind-at[db ts kind] (kind ((:timestamped db) ts))) - From ab7a49ee9ddc6cfbbb595cbd37a33c88decd0b73 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 4 Mar 2014 07:23:25 +0200 Subject: [PATCH 036/102] better namespacing --- functionalDB/src/core/manage.clj | 2 +- functionalDB/src/scenarios/hospital.clj | 35 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 functionalDB/src/scenarios/hospital.clj diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index d921b91d6..d63b5a6a0 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,4 +1,4 @@ -(ns tmpTst (:use core.fdb)) +(ns core.manage (:use core.fdb)) (def all-dbs (atom {})) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj new file mode 100644 index 000000000..52f66e7dd --- /dev/null +++ b/functionalDB/src/scenarios/hospital.clj @@ -0,0 +1,35 @@ +(ns core.tmpTst + [:use core.fdb] + [:require [core.manage :as M] + ]) + + +(def hospital-db (M/get-db-conn "hospital-db")) + + +(def sis (make-entity :test/db-systolic)) +(def dias (make-entity :test/db-diastolic)) +(def temp (make-entity :test/temperature)) +(def person (make-entity :person/patient)) +(def doctor (make-entity :person/doctor)) + +(def pat1 (-> (make-entity :pat1 ) + (add-attr (make-attr :patient/kind :person/patient :REF )) + (add-attr (make-attr :patient/systolic 120 :number )) + (add-attr (make-attr :patient/diastolic 80 :number )) + (add-attr (make-attr :patient/temperature 37 :number )) + (add-attr (make-attr :patient/city "London":string :db/single true)) + ) + + (def pat2 (-> (make-entity :__pat2 ) + (add-attr (make-attr :patient/kind :person/patient :REF )) + (add-attr (make-attr :patient/systolic 110 :number )) + (add-attr (make-attr :patient/diastolic 70 :number )) + (add-attr (make-attr :patient/temperature 38 :number )) + (add-attr (make-attr :patient/city "New York":string :db/single true)) + ) + +;; world setup +(transact hospital-db (add-entity sis) (add-entity dias) (add-entity temp) (add-entity person) (add-entity doctor)) +(transact hospital-db (add-entity pat1) (add-entity pat2)) + From 38d1886a2f1549121148af7a63ef87b9c6f00bd4 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 4 Mar 2014 07:26:35 +0200 Subject: [PATCH 037/102] initial demo scenario --- functionalDB/src/scenarios/hospital.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 52f66e7dd..3d1f1b465 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,4 +1,4 @@ -(ns core.tmpTst +(ns scenarios.hospital [:use core.fdb] [:require [core.manage :as M] ]) @@ -19,7 +19,7 @@ (add-attr (make-attr :patient/diastolic 80 :number )) (add-attr (make-attr :patient/temperature 37 :number )) (add-attr (make-attr :patient/city "London":string :db/single true)) - ) + )) (def pat2 (-> (make-entity :__pat2 ) (add-attr (make-attr :patient/kind :person/patient :REF )) @@ -27,7 +27,7 @@ (add-attr (make-attr :patient/diastolic 70 :number )) (add-attr (make-attr :patient/temperature 38 :number )) (add-attr (make-attr :patient/city "New York":string :db/single true)) - ) + )) ;; world setup (transact hospital-db (add-entity sis) (add-entity dias) (add-entity temp) (add-entity person) (add-entity doctor)) From 8c6c97bc6fb07616dc79a2bfe02a09789c3104a1 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 4 Mar 2014 14:02:35 +0200 Subject: [PATCH 038/102] better management of dbs and resetting of demo db upon creation --- functionalDB/src/core/manage.clj | 16 ++++++++++++++-- functionalDB/src/scenarios/hospital.clj | 5 +++-- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index d63b5a6a0..3c42af37d 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,11 +1,23 @@ +;; management of db and db connections +;; via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection (ns core.manage (:use core.fdb)) -(def all-dbs (atom {})) +(def __ALL-DBS__ (atom {})) (defn _get-db [dbs db-name] (if (db-name dbs ) dbs (assoc dbs db-name (make-db)))) +(defn _reset-db[dbs db-name] + (dissoc dbs db-name)) + +(defn _as-db-name [db-name] (keyword db-name)) + (defn get-db-conn [db-name] - ((keyword db-name) (swap! all-dbs _get-db (keyword db-name)))) + (let [stored-db-name (_as-db-name db-name)] + (stored-db-name (swap! __ALL-DBS__ _get-db stored-db-name)))) + +(defn reset-db-conn [db-name] + (let [stored-db-name (_as-db-name db-name)] + (swap! __ALL-DBS__ _reset-db stored-db-name)) nil) (defn db-from-conn [conn] @conn) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 3d1f1b465..cfd113213 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,9 +1,10 @@ (ns scenarios.hospital [:use core.fdb] - [:require [core.manage :as M] - ]) + [:require [core.manage :as M]]) +(M/reset-db-conn "hospital-db") + (def hospital-db (M/get-db-conn "hospital-db")) From 995480450d6f5c1828704bc1a143091d7616fa98 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 4 Mar 2014 17:31:02 +0200 Subject: [PATCH 039/102] updating make-attr to receive named arguments with proper defaults for the indexing and cardinality. Fixing the issue that operations on attributes with multiple cardinality now accept sets as arguments --- functionalDB/src/core/fdb.clj | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index d4dc0f6f3..a2e3e5a3f 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,4 +1,5 @@ -(ns core.fdb) +(ns core.fdb + [:require [clojure.set :as CS :only (union difference )]]) (defrecord Database [timestamped top-id curr-time]) (defrecord Indices [EAVT VAET AVET]) @@ -13,26 +14,35 @@ ([id] (Entity. id {}))) (defn make-attr - ([name value type ] (make-attr name value type :db/single)) - ([name value type cardinality] (make-attr name value type cardinality false)) - ([name value type cardinality indexed] (with-meta (Attr. name type value -1 -1) {:indexed indexed :cardinality cardinality} ))) + "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguguments. + The type of the attribute may be either :string, :number, :boolean or :REF. If the type is :REF, the value is an id of another entity and indexing of backpointing is maintained. + The named arguments are as follows: + :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. + :cardinality - the cardinality of an attribute, can be either: + :db/single - which means that this attribute can be a single value at any given time (this is the default cardinality) + :db/mutiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): + :db/add - adds a set of values to the currently existing set of values + :db/resetTo - resets the value of this attribute to be the given set of values + :db/remove - removes the given set of values from the attribute's current set of values" + ([name value type ; these ones are required + & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] + (with-meta (Attr. name type value -1 -1) {:indexed indexed :cardinality cardinality} ))) (defn update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" [attr value operation] (cond (= :db/single (:cardinality (meta attr))) (assoc attr :value #{value}) ; now = :db/mutiple (:cardinality (meta attr))) - (= :db/reset-to operation) (assoc attr :value #{value}) - (= :db/add operation) (assoc attr :value (conj (:value attr) value )) - (= :db/remove operation) (assoc attr :value (disj (:value attr) value )) - )) + (= :db/reset-to operation) (assoc attr :value value) + (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) + (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) (defn conj-to-attr "conjoins a value to an attribute, this function can be used either when initializing a new attribute or updating an existing one" [old new operation] (let [new-val (:value new)] (if old (update-attr-value old (:value new) operation) - (update-attr-value new (:value new) :db/reset-to )))) + (update-attr-value new (:value new) :db/reset-to)))) (defn add-attr "adds an attribute to an entity" [ ent attr] @@ -118,7 +128,6 @@ (defn transact-on-db [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] - (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [ initial-indices (:timestamped initial-db) new-indices (last (:timestamped transacted)) From 774ac59a2a37b2ce0291b612c21c27a09bf620c8 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 5 Mar 2014 11:16:48 +0200 Subject: [PATCH 040/102] fixing typo and testing indexed and cardinality using predicate functions --- functionalDB/src/core/fdb.clj | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index a2e3e5a3f..e1fbfeaa1 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -20,7 +20,7 @@ :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. :cardinality - the cardinality of an attribute, can be either: :db/single - which means that this attribute can be a single value at any given time (this is the default cardinality) - :db/mutiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): + :db/multiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): :db/add - adds a set of values to the currently existing set of values :db/resetTo - resets the value of this attribute to be the given set of values :db/remove - removes the given set of values from the attribute's current set of values" @@ -28,11 +28,17 @@ & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] (with-meta (Attr. name type value -1 -1) {:indexed indexed :cardinality cardinality} ))) +(defn single? [attr] + (= :db/single (:cardinality (meta attr)))) + +(defn indexed? [attr] + (:indexed (meta attr))) + (defn update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" [attr value operation] (cond - (= :db/single (:cardinality (meta attr))) (assoc attr :value #{value}) - ; now = :db/mutiple (:cardinality (meta attr))) + (single? attr) (assoc attr :value #{value}) + ; now = :db/multiple (:cardinality (meta attr))) (= :db/reset-to operation) (assoc attr :value value) (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) @@ -98,7 +104,7 @@ ;avet : attr-name -> attr-value -> #{ids of ents} (defn update-avet [old-avet ent operation] - (let [indexed-attrs (filter #(:indexed (meta %)) (vals (:attrs ent))) + (let [indexed-attrs (filter #(indexed? %) (vals (:attrs ent))) update-attr-in-avet-fn (partial update-attr-in-avet ent operation)] (reduce update-attr-in-avet-fn old-avet indexed-attrs))) @@ -186,7 +192,7 @@ ] updated-vaet))) (defn update-avet-for-datom [avet ent-id attr target-attr-val operation] - (if-not (:indexed (meta attr)) + (if-not (indexed? attr) avet (let [old-attr-vals (:value attr) attr-name (:name attr) @@ -204,7 +210,7 @@ (defn update-datom ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) - ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/mutiple) + ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) (let [ new-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name] ) From d4707315886993e0b4e093a1c8bafa0a129f8608 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 5 Mar 2014 11:56:32 +0200 Subject: [PATCH 041/102] few more steps in the example --- functionalDB/src/core/fdb.clj | 3 ++- functionalDB/src/scenarios/hospital.clj | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index e1fbfeaa1..fd65c9435 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -245,7 +245,8 @@ reffing-ids (get-in indices [:VAET ent-id attr-name])] (map #(get-in indices [:EAVT %]) reffing-ids )))) -(defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] +(defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" + [db ent-id attr-name] (loop [res [] ts (:curr-time db)] (if (= -1 ts) (reverse res) (let [attr (attr-at db ent-id attr-name ts)] diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index cfd113213..74e1e0863 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,5 +1,5 @@ (ns scenarios.hospital - [:use core.fdb] + (:use core.fdb) [:require [core.manage :as M]]) @@ -14,12 +14,13 @@ (def person (make-entity :person/patient)) (def doctor (make-entity :person/doctor)) -(def pat1 (-> (make-entity :pat1 ) +(def pat1 (-> (make-entity :__pat1 ) (add-attr (make-attr :patient/kind :person/patient :REF )) (add-attr (make-attr :patient/systolic 120 :number )) (add-attr (make-attr :patient/diastolic 80 :number )) (add-attr (make-attr :patient/temperature 37 :number )) - (add-attr (make-attr :patient/city "London":string :db/single true)) + (add-attr (make-attr :patient/city "London":string :indexed true)) + (add-attr (make-attr :patient/symptoms #{"fever" "cough"}:string :cardinality :db/multiple)) )) (def pat2 (-> (make-entity :__pat2 ) @@ -27,10 +28,13 @@ (add-attr (make-attr :patient/systolic 110 :number )) (add-attr (make-attr :patient/diastolic 70 :number )) (add-attr (make-attr :patient/temperature 38 :number )) - (add-attr (make-attr :patient/city "New York":string :db/single true)) + (add-attr (make-attr :patient/city "New York":string :indexd true)) )) ;; world setup (transact hospital-db (add-entity sis) (add-entity dias) (add-entity temp) (add-entity person) (add-entity doctor)) (transact hospital-db (add-entity pat1) (add-entity pat2)) +(transact hospital-db (update-datom :__pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/add)) +(evolution-of (M/db-from-conn hospital-db) :__pat1 :patient/symptoms) + From 166daed708aee0d07860180143b79e81c9660b30 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 7 Mar 2014 10:37:22 +0200 Subject: [PATCH 042/102] making non-public API functions to be defined using defn-, making names idimatic and better and clearer example --- functionalDB/src/core/fdb.clj | 115 ++++++++++++++---------- functionalDB/src/scenarios/hospital.clj | 54 +++++++---- 2 files changed, 101 insertions(+), 68 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index fd65c9435..958b8ad63 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -4,10 +4,10 @@ (defrecord Database [timestamped top-id curr-time]) (defrecord Indices [EAVT VAET AVET]) (defrecord Entity [id attrs]) -(defrecord Attr [name type value ts prev-ts]) +(defrecord Attr [name value ts prev-ts]) (defn make-db "Create an empty database" [] - (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are REFs, we hold the back-pointing (from the REFFed entity to the REFing entities) + (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are of type :db/ref, we hold the back-pointing (from the REFFed entity to the REFing entities) (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) @@ -15,7 +15,7 @@ (defn make-attr "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguguments. - The type of the attribute may be either :string, :number, :boolean or :REF. If the type is :REF, the value is an id of another entity and indexing of backpointing is maintained. + The type of the attribute may be either :string, :number, :boolean or :db/ref. If the type is :db/ref, the value is an id of another entity and indexing of backpointing is maintained. The named arguments are as follows: :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. :cardinality - the cardinality of an attribute, can be either: @@ -26,15 +26,19 @@ :db/remove - removes the given set of values from the attribute's current set of values" ([name value type ; these ones are required & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] - (with-meta (Attr. name type value -1 -1) {:indexed indexed :cardinality cardinality} ))) + (with-meta (Attr. name value -1 -1) {:type type :indexed indexed :cardinality cardinality} ))) -(defn single? [attr] +(defn- single? [attr] (= :db/single (:cardinality (meta attr)))) -(defn indexed? [attr] +(defn- indexed? [attr] (:indexed (meta attr))) -(defn update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" +(defn- ref? [attr] + (= :db/ref (:type (meta attr)))) + +(defn- update-attr-value + "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" [attr value operation] (cond (single? attr) (assoc attr :value #{value}) @@ -43,31 +47,37 @@ (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) -(defn conj-to-attr "conjoins a value to an attribute, this function can be used either when initializing a new attribute or updating an existing one" +(defn- conj-to-attr + "conjoins a value to an attribute, this function can be used either when initializing a new attribute or updating an existing one" [old new operation] (let [new-val (:value new)] (if old (update-attr-value old (:value new) operation) (update-attr-value new (:value new) :db/reset-to)))) -(defn add-attr "adds an attribute to an entity" +(defn add-attr + "adds an attribute to an entity" [ ent attr] (let [attr-id (keyword (:name attr)) existing-attr (get-in ent [:attrs attr-id]) updated-attr (conj-to-attr existing-attr attr :db/add)] (assoc-in ent [:attrs attr-id ] updated-attr))) -(defn next-ts [db] "returns the next timestamp of the given database" +(defn- next-ts [db] + "returns the next timestamp of the given database" (inc (:curr-time db))) -(defn nextId [db ent] "returns a pair composed of the id to use for the given entity and the next free running id in the database" +(defn- next-id + "returns a pair composed of the id to use for the given entity and the next free running id in the database" + [db ent] (let [top-id (:top-id db) - entId (:id ent) + ent-id (:id ent) inceased-id (inc top-id) - [idToUse nextTop] (if (= entId :db/no-id-yet) [(keyword (str inceased-id)) inceased-id] [entId top-id])] - [idToUse nextTop])) + [id-to-use next-top] (if (= ent-id :db/no-id-yet) [(keyword (str inceased-id)) inceased-id] [ent-id top-id])] + [id-to-use next-top])) -(defn update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" +(defn- update-creation-ts + "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent tsVal] (let [ks (keys (:attrs ent)) vls (vals (:attrs ent)) @@ -78,7 +88,8 @@ ; VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} ; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) ; can be used to know who REFs a specific entity -(defn update-ref-in-vaet "adding an entry to th VAET index" +(defn- update-ref-in-vaet + "adding an entry to th VAET index" [ent operation vaet attr] (let [reffed-id (first (:value attr)) attr-name (:name attr) @@ -89,7 +100,7 @@ ; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} ; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) ; can be used to know who are the entities that have a specific attribute with a specific value -(defn update-attr-in-avet "" +(defn- update-attr-in-avet "" [ent operation avet attr] (let [attr-name (:name attr) attr-value (first (:value attr)) ; a set @@ -97,20 +108,20 @@ updated-entities-set (operation curr-entities-set (:id ent)) ] (assoc-in avet [attr-name attr-value] updated-entities-set))) -(defn update-vaet[old-vaet ent operation] - (let [reffingAttrs (filter #(= :REF (:type %)) (vals (:attrs ent))) +(defn- update-vaet[old-vaet ent operation] + (let [reffingAttrs (filter #(ref? %) (vals (:attrs ent))) add-ref (partial update-ref-in-vaet ent operation)] (reduce add-ref old-vaet reffingAttrs))) ;avet : attr-name -> attr-value -> #{ids of ents} -(defn update-avet [old-avet ent operation] +(defn- update-avet [old-avet ent operation] (let [indexed-attrs (filter #(indexed? %) (vals (:attrs ent))) update-attr-in-avet-fn (partial update-attr-in-avet ent operation)] (reduce update-attr-in-avet-fn old-avet indexed-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity[db ent] - (let [[ent-id next-top] (nextId db ent) + (let [[ent-id next-top] (next-id db ent) new-ts (next-ts db) indices (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) @@ -142,45 +153,43 @@ :top-id (:top-id transacted))] res)))) -(defmacro transact_ [db op & txs] +(defmacro _transact [db op & txs] (when txs - (loop [[frst-tx# & rst-tx#] txs res# [op db 'transact-on-db] accum-txs# []] - (if frst-tx# (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) - (list* (conj res# accum-txs#)))))) + (loop [[frst-tx# & rst-tx#] txs res# [op db 'transact-on-db] accum-txs# []] + (if frst-tx# + (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) + (list* (conj res# accum-txs#)))))) -(defn _what-if [ db f txs] (f db txs)) +(defn- _what-if [ db f txs] (f db txs)) (defmacro what-if [db & txs] - `(transact_ ~db _what-if ~@txs)) + `(_transact ~db _what-if ~@txs)) (defmacro transact [db & txs] - `(transact_ ~db swap! ~@txs)) + `(_transact ~db swap! ~@txs)) ;;;;;;;;;;;;;;;;;;;;;;; datom updates -(defn remove-entry-from-index [val-to-remove index path ] +(defn- remove-entry-from-index [val-to-remove index path ] (let [ old-entries-set (get-in index path )] - (assoc-in index path (disj old-entries-set val-to-remove)) - )) + (assoc-in index path (disj old-entries-set val-to-remove)))) -(defn remove-entries-from-index [index paths val-to-remove operation] +(defn- remove-entries-from-index [index paths val-to-remove operation] (if (= operation :db/add) index - (let [ remover (partial remove-entry-from-index val-to-remove) ] - (reduce remover index (operation paths)) - ))) + (let [ remover (partial remove-entry-from-index val-to-remove)] + (reduce remover index (operation paths))))) -(defn add-entry-to-index [index path val-to-add operation] +(defn- add-entry-to-index [index path val-to-add operation] (if (= operation :db/remove) index (let [to-be-updated-set (get-in index path #{})] (assoc-in index path (conj to-be-updated-set val-to-add))))) -(defn update-vaet-for-datom [vaet ent-id attr target-ref-id operation] - (if (not= :REF (:type attr )) +(defn- update-vaet-for-datom [vaet ent-id attr target-ref-id operation] + (if-not (ref? attr) vaet - (let [ - old-ref-ids (:value attr) + (let [old-ref-ids (:value attr) attr-name (:name attr) paths-in-case-of-replace (map (fn[i][i attr-name]) old-ref-ids) paths-in-case-of-remove [[target-ref-id attr-name]] @@ -191,7 +200,7 @@ updated-vaet (add-entry-to-index cleaned-vaet [target-ref-id attr-name] ent-id operation) ] updated-vaet))) -(defn update-avet-for-datom [avet ent-id attr target-attr-val operation] +(defn- update-avet-for-datom [avet ent-id attr target-attr-val operation] (if-not (indexed? attr) avet (let [old-attr-vals (:value attr) @@ -205,7 +214,7 @@ updated-avet (add-entry-to-index cleaned-avet [attr-name target-attr-val] ent-id operation) ] updated-avet ))) -(defn update-eavt-for-datom [eavt ent-id new-attr] +(defn- update-eavt-for-datom [eavt ent-id new-attr] (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) (defn update-datom @@ -214,7 +223,7 @@ (let [ new-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name] ) - real-new-val (if (= :REF (:type attr)) (:id new-val) new-val) + real-new-val (if (ref? attr) (:id new-val) new-val) updated-timed-attr (assoc attr :ts new-ts :prev-ts ( :ts attr)) updated-attr (conj-to-attr updated-timed-attr {:value real-new-val} operation) new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) @@ -229,23 +238,28 @@ (defn entity-at ([db ent-id ts] ((keyword ent-id) ((:timestamped db) ts))) ([db ent-id] (entity-at db ent-id (:curr-time db))) ) -(defn attr-at "The attribute of an entity at a given time (defaults to recent time)" +(defn attr-at + "The attribute of an entity at a given time (defaults to recent time)" ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] (let [indices ((:timestamped db) ts)] (get-in indices [:EAVT ent-id :attrs attr-name])))) -(defn value-of-at "value of a datom at a given time, if no time is provided, we default to the most recent value" +(defn value-of-at + "value of a datom at a given time, if no time is provided, we default to the most recent value" ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) -(defn relates-to-as "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is REF and the value was ent-id), all this at a given time" - ([db ent-id attr-name] (relates-to-as db ent-id attr-name (:curr-time db))) +(defn ref-to-as + "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is :db/ref + and the value was ent-id), all this at a given time" + ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] (let [indices ((:timestamped db) ts) reffing-ids (get-in indices [:VAET ent-id attr-name])] (map #(get-in indices [:EAVT %]) reffing-ids )))) -(defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" +(defn evolution-of + "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] (loop [res [] ts (:curr-time db)] (if (= -1 ts) (reverse res) @@ -256,4 +270,7 @@ (let [indices-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped indices-before :curr-time ts))) -(defn ind-at[db ts kind] (kind ((:timestamped db) ts))) +(defn ind-at + "inspecting a specific index at a given time. The kind argument may be of of these: :AVET :EAVT :VAET " + [db ts kind] + (kind ((:timestamped db) ts))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 74e1e0863..162e9a527 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -14,27 +14,43 @@ (def person (make-entity :person/patient)) (def doctor (make-entity :person/doctor)) -(def pat1 (-> (make-entity :__pat1 ) - (add-attr (make-attr :patient/kind :person/patient :REF )) - (add-attr (make-attr :patient/systolic 120 :number )) - (add-attr (make-attr :patient/diastolic 80 :number )) - (add-attr (make-attr :patient/temperature 37 :number )) - (add-attr (make-attr :patient/city "London":string :indexed true)) - (add-attr (make-attr :patient/symptoms #{"fever" "cough"}:string :cardinality :db/multiple)) - )) - - (def pat2 (-> (make-entity :__pat2 ) - (add-attr (make-attr :patient/kind :person/patient :REF )) - (add-attr (make-attr :patient/systolic 110 :number )) - (add-attr (make-attr :patient/diastolic 70 :number )) - (add-attr (make-attr :patient/temperature 38 :number )) - (add-attr (make-attr :patient/city "New York":string :indexd true)) - )) +;; (def pat1 (-> (make-entity :__pat1 ) +;; (add-attr (make-attr :patient/kind :person/patient :db/ref )) +;; (add-attr (make-attr :patient/systolic 120 :number )) +;; (add-attr (make-attr :patient/diastolic 80 :number )) +;; (add-attr (make-attr :patient/temperature 37 :number )) +;; (add-attr (make-attr :patient/city "London":string :indexed true)) +;; (add-attr (make-attr :patient/symptoms #{"fever" "cough"}:string :cardinality :db/multiple)) +;; )) + +;; (def pat2 (-> (make-entity :__pat2 ) +;; (add-attr (make-attr :patient/kind :person/patient :db/ref )) +;; (add-attr (make-attr :patient/systolic 110 :number )) +;; (add-attr (make-attr :patient/diastolic 70 :number )) +;; (add-attr (make-attr :patient/temperature 38 :number )) +;; (add-attr (make-attr :patient/city "New York":string :indexd true)) +;; )) ;; world setup + +(defn make-patient[id address symptoms] + (-> (make-entity id) + (add-attr (make-attr :patient/kind :person/patient :db/ref)) + (add-attr (make-attr :patient/city address :string :indexed true)) + (add-attr (make-attr :patient/tests [] :db/ref :indexed true :cardinality :db/multiple)) + (add-attr (make-attr :patient/symptoms (set symptoms) :string :cardinality :db/multiple)))) + +(defn add-patient [id address symptoms] + (transact hospital-db (add-entity (make-patient id address symptoms)))) + + (transact hospital-db (add-entity sis) (add-entity dias) (add-entity temp) (add-entity person) (add-entity doctor)) -(transact hospital-db (add-entity pat1) (add-entity pat2)) -(transact hospital-db (update-datom :__pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/add)) -(evolution-of (M/db-from-conn hospital-db) :__pat1 :patient/symptoms) +@hospital-db +(add-patient :pat1 "London" ["fever" "cough"] ) +;(add-patient :pat2 "New York" ["Tremor" "dizziness"]) + +;(transact hospital-db (add-entity pat1) (add-entity pat2)) +(transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/add)) +(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) From 19560dcf514c508e0b1ae07c2bf8579c18fb5cf6 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 7 Mar 2014 10:38:10 +0200 Subject: [PATCH 043/102] remove commented out line --- functionalDB/src/scenarios/hospital.clj | 20 +------------------- 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 162e9a527..83cc1e479 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -14,23 +14,6 @@ (def person (make-entity :person/patient)) (def doctor (make-entity :person/doctor)) -;; (def pat1 (-> (make-entity :__pat1 ) -;; (add-attr (make-attr :patient/kind :person/patient :db/ref )) -;; (add-attr (make-attr :patient/systolic 120 :number )) -;; (add-attr (make-attr :patient/diastolic 80 :number )) -;; (add-attr (make-attr :patient/temperature 37 :number )) -;; (add-attr (make-attr :patient/city "London":string :indexed true)) -;; (add-attr (make-attr :patient/symptoms #{"fever" "cough"}:string :cardinality :db/multiple)) -;; )) - -;; (def pat2 (-> (make-entity :__pat2 ) -;; (add-attr (make-attr :patient/kind :person/patient :db/ref )) -;; (add-attr (make-attr :patient/systolic 110 :number )) -;; (add-attr (make-attr :patient/diastolic 70 :number )) -;; (add-attr (make-attr :patient/temperature 38 :number )) -;; (add-attr (make-attr :patient/city "New York":string :indexd true)) -;; )) - ;; world setup (defn make-patient[id address symptoms] @@ -47,9 +30,8 @@ (transact hospital-db (add-entity sis) (add-entity dias) (add-entity temp) (add-entity person) (add-entity doctor)) @hospital-db (add-patient :pat1 "London" ["fever" "cough"] ) -;(add-patient :pat2 "New York" ["Tremor" "dizziness"]) +(add-patient :pat2 "New York" ["Tremor" "dizziness"]) -;(transact hospital-db (add-entity pat1) (add-entity pat2)) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/add)) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) From 1a82164bb2132f9047610801eb81d16ffd05ae40 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 9 Mar 2014 19:16:32 +0200 Subject: [PATCH 044/102] fixing the problem of indexing multiple refs, removing duplications --- functionalDB/src/core/fdb.clj | 66 ++++++++++++------------- functionalDB/src/scenarios/hospital.clj | 42 +++++++++------- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 958b8ad63..9ad92900a 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -91,11 +91,15 @@ (defn- update-ref-in-vaet "adding an entry to th VAET index" [ent operation vaet attr] - (let [reffed-id (first (:value attr)) + ( + let [reffed-id (first (:value attr)) attr-name (:name attr) back-reffing-set (get-in vaet [reffed-id attr-name] #{} ) new-back-reffing-set (operation back-reffing-set (:id ent)) - ] (assoc-in vaet [reffed-id attr-name] new-back-reffing-set))) + ] + (if (nil? reffed-id) + vaet + (assoc-in vaet [reffed-id attr-name] new-back-reffing-set)))) ; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} ; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) @@ -106,7 +110,10 @@ attr-value (first (:value attr)) ; a set curr-entities-set (get-in avet [attr-name attr-value] #{}) updated-entities-set (operation curr-entities-set (:id ent)) - ] (assoc-in avet [attr-name attr-value] updated-entities-set))) + ] + (if (nil? attr-value) + avet + (assoc-in avet [attr-name attr-value] updated-entities-set)))) (defn- update-vaet[old-vaet ent operation] (let [reffingAttrs (filter #(ref? %) (vals (:attrs ent))) @@ -120,7 +127,7 @@ (reduce update-attr-in-avet-fn old-avet indexed-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity[db ent] +(defn add-entity [db ent] (let [[ent-id next-top] (next-id db ent) new-ts (next-ts db) indices (last (:timestamped db)) @@ -132,6 +139,9 @@ ](assoc db :timestamped (conj (:timestamped db) new-indices) :top-id next-top))) +(defn add-entities [db ents-seq] + (reduce add-entity db ents-seq)) + (defn remove-entity[db ent] (let [ent-id (:id ent) indices (last (:timestamped db)) @@ -186,33 +196,24 @@ (let [to-be-updated-set (get-in index path #{})] (assoc-in index path (conj to-be-updated-set val-to-add))))) -(defn- update-vaet-for-datom [vaet ent-id attr target-ref-id operation] - (if-not (ref? attr) - vaet - (let [old-ref-ids (:value attr) +(defn- av [attr vl] [attr vl] ) +(defn- va [attr vl] [vl attr] ) + +(defn- update-index-for-datom[index ent-id attr target-val operation order-fn update-required-pred] +(if-not (update-required-pred attr) + index + (let [old-vals (:value attr) attr-name (:name attr) - paths-in-case-of-replace (map (fn[i][i attr-name]) old-ref-ids) - paths-in-case-of-remove [[target-ref-id attr-name]] - paths {:db/remove paths-in-case-of-remove :db/reset-to paths-in-case-of-replace} + paths-for-replace (map (fn[i](order-fn attr-name i)) old-vals) ; replace means that we need to remove all the old items, and add the new item + paths-for-remove [(order-fn attr-name target-val)] ; remove means that the item that we received is the one to be removed + paths {:db/remove paths-for-remove :db/reset-to paths-for-replace} ;; removing old values - cleaned-vaet (remove-entries-from-index vaet paths ent-id operation) + cleaned-index (remove-entries-from-index index paths ent-id operation) ;; adding new value - updated-vaet (add-entry-to-index cleaned-vaet [target-ref-id attr-name] ent-id operation) - ] updated-vaet))) - -(defn- update-avet-for-datom [avet ent-id attr target-attr-val operation] - (if-not (indexed? attr) - avet - (let [old-attr-vals (:value attr) - attr-name (:name attr) - paths-in-case-of-replace (map (fn[i][attr-name i]) old-attr-vals) - paths-in-case-of-remove [[attr-name target-attr-val]] - paths {:db/remove paths-in-case-of-remove :db/reset-to paths-in-case-of-replace} - ;; removing old value - cleaned-avet (remove-entries-from-index avet paths ent-id operation) - ;; adding new value - updated-avet (add-entry-to-index cleaned-avet [attr-name target-attr-val] ent-id operation) - ] updated-avet ))) + colled-target-val (if (coll? target-val) target-val [ target-val]) + add-func (fn [ind vl] (add-entry-to-index ind (order-fn attr-name vl) ent-id operation)) + updated-index (reduce add-func cleaned-index colled-target-val) + ] updated-index))) (defn- update-eavt-for-datom [eavt ent-id new-attr] (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) @@ -223,19 +224,18 @@ (let [ new-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name] ) - real-new-val (if (ref? attr) (:id new-val) new-val) updated-timed-attr (assoc attr :ts new-ts :prev-ts ( :ts attr)) - updated-attr (conj-to-attr updated-timed-attr {:value real-new-val} operation) + updated-attr (conj-to-attr updated-timed-attr {:value new-val} operation) new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) - new-vaet (update-vaet-for-datom (:VAET indices) ent-id attr real-new-val operation) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - new-avet (update-avet-for-datom (:AVET indices) ent-id attr real-new-val operation) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation va ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index fully-updated-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet) new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) ]new-db))) ;;;;;;;;;;;;;;;;;;;;;;; queries -(defn entity-at ([db ent-id ts] ((keyword ent-id) ((:timestamped db) ts))) +(defn entity-at ([db ent-id ts] (ent-id ((:timestamped db) ts))) ([db ent-id] (entity-at db ent-id (:curr-time db))) ) (defn attr-at diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 83cc1e479..d3e060444 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -2,37 +2,43 @@ (:use core.fdb) [:require [core.manage :as M]]) +(def db-name "hos9") +(M/reset-db-conn db-name) -(M/reset-db-conn "hospital-db") +(def hospital-db (M/get-db-conn db-name)) -(def hospital-db (M/get-db-conn "hospital-db")) - - -(def sis (make-entity :test/db-systolic)) -(def dias (make-entity :test/db-diastolic)) -(def temp (make-entity :test/temperature)) -(def person (make-entity :person/patient)) -(def doctor (make-entity :person/doctor)) - -;; world setup +(def basic-kinds [:test/bp-systolic :test/bp-diastolic :test/temperature :person/patient :person/doctor ] ) (defn make-patient[id address symptoms] (-> (make-entity id) (add-attr (make-attr :patient/kind :person/patient :db/ref)) - (add-attr (make-attr :patient/city address :string :indexed true)) - (add-attr (make-attr :patient/tests [] :db/ref :indexed true :cardinality :db/multiple)) + (add-attr (make-attr :patient/city address :string )) + (add-attr (make-attr :patient/tests #{} :db/ref :indexed true :cardinality :db/multiple)) (add-attr (make-attr :patient/symptoms (set symptoms) :string :cardinality :db/multiple)))) +(defn make-test [t-id tests-map types indexedPred] + (let [ent (make-entity t-id)] + (reduce #(add-attr %1 (make-attr (first %2) ;attr-name + (second %2) ; attr value + (get types (first %2) :number) ; attr type + :indexed (indexedPred (first %2)) ));indexed + ent tests-map))) + (defn add-patient [id address symptoms] (transact hospital-db (add-entity (make-patient id address symptoms)))) +(defn add-test-results-to-patient [pat-id test-result] + (let [test-id (:id test-result) ] + (transact hospital-db (add-entity test-result)) + (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) -(transact hospital-db (add-entity sis) (add-entity dias) (add-entity temp) (add-entity person) (add-entity doctor)) -@hospital-db + +;; world setup +(transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) (add-patient :pat1 "London" ["fever" "cough"] ) -(add-patient :pat2 "New York" ["Tremor" "dizziness"]) +(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) +(add-test-results-to-patient :pat1 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/add)) -(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) - +(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) From 88cf998697dc0affd81369f319c298d58cd46dc7 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 10 Mar 2014 13:22:27 +0200 Subject: [PATCH 045/102] refactoring the update process --- functionalDB/src/core/fdb.clj | 45 ++++++++++++++----------- functionalDB/src/scenarios/hospital.clj | 4 +-- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 9ad92900a..f33682ffb 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -22,7 +22,7 @@ :db/single - which means that this attribute can be a single value at any given time (this is the default cardinality) :db/multiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): :db/add - adds a set of values to the currently existing set of values - :db/resetTo - resets the value of this attribute to be the given set of values + :db/reset-to - resets the value of this attribute to be the given set of values :db/remove - removes the given set of values from the attribute's current set of values" ([name value type ; these ones are required & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] @@ -42,7 +42,7 @@ [attr value operation] (cond (single? attr) (assoc attr :value #{value}) - ; now = :db/multiple (:cardinality (meta attr))) + ; now we're talking about an attribute of multiple values (= :db/reset-to operation) (assoc attr :value value) (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) @@ -57,7 +57,7 @@ (defn add-attr "adds an attribute to an entity" - [ ent attr] + [ent attr] (let [attr-id (keyword (:name attr)) existing-attr (get-in ent [:attrs attr-id]) updated-attr (conj-to-attr existing-attr attr :db/add)] @@ -91,20 +91,18 @@ (defn- update-ref-in-vaet "adding an entry to th VAET index" [ent operation vaet attr] - ( - let [reffed-id (first (:value attr)) + (let [reffed-id (first (:value attr)) attr-name (:name attr) back-reffing-set (get-in vaet [reffed-id attr-name] #{} ) - new-back-reffing-set (operation back-reffing-set (:id ent)) - ] - (if (nil? reffed-id) + new-back-reffing-set (operation back-reffing-set (:id ent))] + (if (nil? reffed-id) vaet (assoc-in vaet [reffed-id attr-name] new-back-reffing-set)))) ; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} ; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) ; can be used to know who are the entities that have a specific attribute with a specific value -(defn- update-attr-in-avet "" +(defn- update-attr-in-avet [ent operation avet attr] (let [attr-name (:name attr) attr-value (first (:value attr)) ; a set @@ -170,7 +168,7 @@ (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) (list* (conj res# accum-txs#)))))) -(defn- _what-if [ db f txs] (f db txs)) +(defn _what-if [ db f txs] (f db txs)) (defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) @@ -199,8 +197,9 @@ (defn- av [attr vl] [attr vl] ) (defn- va [attr vl] [vl attr] ) -(defn- update-index-for-datom[index ent-id attr target-val operation order-fn update-required-pred] -(if-not (update-required-pred attr) +(defn- update-index-for-datom + [index ent-id attr target-val operation order-fn update-required-pred] + (if-not (update-required-pred attr) index (let [old-vals (:value attr) attr-name (:name attr) @@ -218,19 +217,25 @@ (defn- update-eavt-for-datom [eavt ent-id new-attr] (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) +(defn- update-attr [attr new-val new-ts operation] + (let [updated-timed-attr (assoc attr :ts new-ts :prev-ts ( :ts attr))] + (conj-to-attr updated-timed-attr {:value new-val} operation))) + +(defn- update-indices [indices ent-id attr updated-attr new-val operation] + (let [new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) + new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation va ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + ](assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) + (defn update-datom ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) (let [ new-ts (next-ts db) indices (last (:timestamped db)) - attr (get-in indices [:EAVT ent-id :attrs att-name] ) - updated-timed-attr (assoc attr :ts new-ts :prev-ts ( :ts attr)) - updated-attr (conj-to-attr updated-timed-attr {:value new-val} operation) - new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) - new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation va ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - fully-updated-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet) - new-db (assoc db :timestamped (conj (:timestamped db) fully-updated-indices)) + attr (get-in indices [:EAVT ent-id :attrs att-name]) + updated-attr (update-attr attr new-val new-ts operation) + fully-updated-indices (update-indices indices ent-id attr updated-attr new-val operation) + new-db (update-in db [:timestamped] conj fully-updated-indices) ]new-db))) ;;;;;;;;;;;;;;;;;;;;;;; queries diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index d3e060444..25fd65439 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -32,13 +32,13 @@ (transact hospital-db (add-entity test-result)) (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) - ;; world setup (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) (add-patient :pat1 "London" ["fever" "cough"] ) (add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) (add-test-results-to-patient :pat1 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) -(transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/add)) +(transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) +@hospital-db From 796d4861af00e26744b921f6b43b452607a77316 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 10 Mar 2014 14:33:54 +0200 Subject: [PATCH 046/102] bug fix in the evolution calculation --- functionalDB/src/core/fdb.clj | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index f33682ffb..6fce30663 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -230,10 +230,10 @@ (defn update-datom ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) - (let [ new-ts (next-ts db) + (let [update-ts (next-ts db) indices (last (:timestamped db)) attr (get-in indices [:EAVT ent-id :attrs att-name]) - updated-attr (update-attr attr new-val new-ts operation) + updated-attr (update-attr attr new-val update-ts operation) fully-updated-indices (update-indices indices ent-id attr updated-attr new-val operation) new-db (update-in db [:timestamped] conj fully-updated-indices) ]new-db))) @@ -269,7 +269,7 @@ (loop [res [] ts (:curr-time db)] (if (= -1 ts) (reverse res) (let [attr (attr-at db ent-id attr-name ts)] - (recur (conj res {ts (:value attr)}) (:prev-ts attr)))))) + (recur (conj res {(:ts attr) (:value attr)}) (:prev-ts attr)))))) (defn db-before [db ts] (let [indices-before (subvec (:timestamped db) 0 ts )] From 461a618fc85902b269128952ad0d11f5afdcad63 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 11 Mar 2014 22:45:31 +0200 Subject: [PATCH 047/102] refactoring and bug fixes related to removal of multiple values entries --- functionalDB/src/core/fdb.clj | 191 ++++++++++++------------ functionalDB/src/scenarios/hospital.clj | 9 +- 2 files changed, 100 insertions(+), 100 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 6fce30663..76e85ef3e 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -15,7 +15,7 @@ (defn make-attr "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguguments. - The type of the attribute may be either :string, :number, :boolean or :db/ref. If the type is :db/ref, the value is an id of another entity and indexing of backpointing is maintained. + The type of the attribute may be either :string, :number, :boolean or :db/ref . If the type is :db/ref, the value is an id of another entity and indexing of backpointing is maintained. The named arguments are as follows: :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. :cardinality - the cardinality of an attribute, can be either: @@ -28,6 +28,8 @@ & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] (with-meta (Attr. name value -1 -1) {:type type :indexed indexed :cardinality cardinality} ))) +(defn collify [x] (if (coll? x) x [x])) + (defn- single? [attr] (= :db/single (:cardinality (meta attr)))) @@ -37,6 +39,20 @@ (defn- ref? [attr] (= :db/ref (:type (meta attr)))) +(defn entity-at ([db ent-id ts] (ent-id ((:timestamped db) ts))) + ([db ent-id] (entity-at db ent-id (:curr-time db))) ) + +(defn attr-at + "The attribute of an entity at a given time (defaults to recent time)" + ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name ts] + (let [indices ((:timestamped db) ts)] (get-in indices [:EAVT ent-id :attrs attr-name])))) + +(defn value-of-at + "value of a datom at a given time, if no time is provided, we default to the most recent value" + ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) + ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) + (defn- update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" [attr value operation] @@ -47,21 +63,14 @@ (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) -(defn- conj-to-attr - "conjoins a value to an attribute, this function can be used either when initializing a new attribute or updating an existing one" - [old new operation] - (let [new-val (:value new)] - (if old - (update-attr-value old (:value new) operation) - (update-attr-value new (:value new) :db/reset-to)))) - (defn add-attr "adds an attribute to an entity" [ent attr] - (let [attr-id (keyword (:name attr)) - existing-attr (get-in ent [:attrs attr-id]) - updated-attr (conj-to-attr existing-attr attr :db/add)] - (assoc-in ent [:attrs attr-id ] updated-attr))) + (let [attr-id (keyword (:name attr))] + (assoc-in ent [:attrs attr-id] attr))) + +(defn- av [attr vl] [attr vl] ) +(defn- va [attr vl] [vl attr] ) (defn- next-ts [db] "returns the next timestamp of the given database" @@ -85,44 +94,50 @@ updatedAttrs (zipmap ks updatedAttrsVals) ](assoc ent :attrs updatedAttrs))) +(defn- remove-entry-from-index [val-to-remove index path ] + (let [ old-entries-set (get-in index path )] + (assoc-in index path (disj old-entries-set val-to-remove)))) + +(defn- remove-entries-from-index [index attr-name ent-id order-fn path-values operation] + (if (= operation :db/add) + index + (let [paths (map #(order-fn attr-name %) path-values) + remover (partial remove-entry-from-index ent-id)] + (reduce remover index paths)))) + ; VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} ; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) ; can be used to know who REFs a specific entity -(defn- update-ref-in-vaet - "adding an entry to th VAET index" - [ent operation vaet attr] - (let [reffed-id (first (:value attr)) - attr-name (:name attr) - back-reffing-set (get-in vaet [reffed-id attr-name] #{} ) - new-back-reffing-set (operation back-reffing-set (:id ent))] - (if (nil? reffed-id) - vaet - (assoc-in vaet [reffed-id attr-name] new-back-reffing-set)))) - -; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} -; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) -; can be used to know who are the entities that have a specific attribute with a specific value -(defn- update-attr-in-avet - [ent operation avet attr] - (let [attr-name (:name attr) - attr-value (first (:value attr)) ; a set - curr-entities-set (get-in avet [attr-name attr-value] #{}) - updated-entities-set (operation curr-entities-set (:id ent)) - ] - (if (nil? attr-value) - avet - (assoc-in avet [attr-name attr-value] updated-entities-set)))) - -(defn- update-vaet[old-vaet ent operation] - (let [reffingAttrs (filter #(ref? %) (vals (:attrs ent))) - add-ref (partial update-ref-in-vaet ent operation)] - (reduce add-ref old-vaet reffingAttrs))) - -;avet : attr-name -> attr-value -> #{ids of ents} -(defn- update-avet [old-avet ent operation] - (let [indexed-attrs (filter #(indexed? %) (vals (:attrs ent))) - update-attr-in-avet-fn (partial update-attr-in-avet ent operation)] - (reduce update-attr-in-avet-fn old-avet indexed-attrs))) + + +;; ; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} +;; ; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) +;; ; can be used to know who are the entities that have a specific attribute with a specific value + +(defn- add-entry-to-index [index path val-to-add operation] + (if (= operation :db/remove) + index + (let [to-be-updated-set (get-in index path #{})] + (assoc-in index path (conj to-be-updated-set val-to-add))))) + +(defn- update-attr-in-index [index ent-id attr-name order-fn target-val operation] + (let [ colled-target-val (collify target-val) + add-entry-fn (fn[indx vl] (add-entry-to-index indx (order-fn attr-name vl) ent-id operation)) + ] (reduce add-entry-fn index colled-target-val))) + +(defn- add-entity-to-index [index ent order-fn filtering-pred] + (let [ent-id (:id ent) + all-attrs (vals (:attrs ent)) + relevant-attrs (filter #(filtering-pred %) all-attrs ) + add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:value attr) order-fn (:value attr) :db/add)) + ] (reduce add-in-index-fn index relevant-attrs))) + +(defn- remove-entity-from-index[index ent order-fn filtering-pred] + (let [ent-id (:id ent) + all-attrs (vals (:attrs ent)) + relevant-attrs (filter #(filtering-pred %) all-attrs ) + remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:value attr) ent-id order-fn (collify (:value attr)) :db/remove)) + ] (reduce remove-from-index-fn index relevant-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity [db ent] @@ -131,8 +146,8 @@ indices (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) new-eavt (assoc (:EAVT indices) ent-id (update-creation-ts fixed-ent new-ts) ) - new-vaet (update-vaet (:VAET indices) fixed-ent conj) - new-avet (update-avet (:AVET indices) fixed-ent conj) + new-vaet (add-entity-to-index (:VAET indices) ent va ref?) + new-avet (add-entity-to-index (:AVET indices) ent av indexed?) new-indices (assoc indices :VAET new-vaet :EAVT new-eavt :AVET new-avet ) ](assoc db :timestamped (conj (:timestamped db) new-indices) :top-id next-top))) @@ -140,11 +155,13 @@ (defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) -(defn remove-entity[db ent] - (let [ent-id (:id ent) +(defn remove-entity[db ent-id] + (let [ent (entity-at db ent-id) indices (last (:timestamped db)) - vaet (update-vaet (:VAET indices) ent disj) - avet (update-avet (:AVET indices) ent disj) + vaet (remove-entity-from-index (:VAET indices) ent va ref? ) + ;vaet (update-vaet (:VAET indices) ent disj) + avet (remove-entity-from-index (:AVET indices) ent av indexed?) + ;avet (update-avet (:AVET indices) ent disj) new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet) @@ -178,24 +195,19 @@ ;;;;;;;;;;;;;;;;;;;;;;; datom updates -(defn- remove-entry-from-index [val-to-remove index path ] - (let [ old-entries-set (get-in index path )] - (assoc-in index path (disj old-entries-set val-to-remove)))) - -(defn- remove-entries-from-index [index paths val-to-remove operation] - (if (= operation :db/add) - index - (let [ remover (partial remove-entry-from-index val-to-remove)] - (reduce remover index (operation paths))))) +(defn- update-attr-modification-time [attr new-ts] + (assoc attr :ts new-ts :prev-ts ( :ts attr))) -(defn- add-entry-to-index [index path val-to-add operation] - (if (= operation :db/remove) - index - (let [to-be-updated-set (get-in index path #{})] - (assoc-in index path (conj to-be-updated-set val-to-add))))) +(defn- update-attr [attr new-val new-ts operation] + (-> attr + (update-attr-modification-time new-ts) + (update-attr-value new-val operation))) -(defn- av [attr vl] [attr vl] ) -(defn- va [attr vl] [vl attr] ) +(defn- remove-path-values [old-vals target-val operation] + (cond + (= operation :db/add) [] + (= operation :db/reset-to) old-vals + (= operation :db/remove) (collify target-val))) (defn- update-index-for-datom [index ent-id attr target-val operation order-fn update-required-pred] @@ -203,29 +215,21 @@ index (let [old-vals (:value attr) attr-name (:name attr) - paths-for-replace (map (fn[i](order-fn attr-name i)) old-vals) ; replace means that we need to remove all the old items, and add the new item - paths-for-remove [(order-fn attr-name target-val)] ; remove means that the item that we received is the one to be removed - paths {:db/remove paths-for-remove :db/reset-to paths-for-replace} - ;; removing old values - cleaned-index (remove-entries-from-index index paths ent-id operation) - ;; adding new value - colled-target-val (if (coll? target-val) target-val [ target-val]) - add-func (fn [ind vl] (add-entry-to-index ind (order-fn attr-name vl) ent-id operation)) - updated-index (reduce add-func cleaned-index colled-target-val) + remove-paths (remove-path-values old-vals target-val operation) + cleaned-index (remove-entries-from-index index attr-name ent-id order-fn remove-paths operation) + updated-index (update-attr-in-index cleaned-index ent-id attr-name order-fn target-val operation) ] updated-index))) (defn- update-eavt-for-datom [eavt ent-id new-attr] (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) -(defn- update-attr [attr new-val new-ts operation] - (let [updated-timed-attr (assoc attr :ts new-ts :prev-ts ( :ts attr))] - (conj-to-attr updated-timed-attr {:value new-val} operation))) - (defn- update-indices [indices ent-id attr updated-attr new-val operation] + (let [new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation va ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - ](assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) + ; x (println new-eavt) + new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + ] (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) (defn update-datom ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) @@ -240,19 +244,6 @@ ;;;;;;;;;;;;;;;;;;;;;;; queries -(defn entity-at ([db ent-id ts] (ent-id ((:timestamped db) ts))) - ([db ent-id] (entity-at db ent-id (:curr-time db))) ) - -(defn attr-at - "The attribute of an entity at a given time (defaults to recent time)" - ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) - ([db ent-id attr-name ts] - (let [indices ((:timestamped db) ts)] (get-in indices [:EAVT ent-id :attrs attr-name])))) - -(defn value-of-at - "value of a datom at a given time, if no time is provided, we default to the most recent value" - ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) - ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) (defn ref-to-as "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is :db/ref @@ -271,7 +262,9 @@ (let [attr (attr-at db ent-id attr-name ts)] (recur (conj res {(:ts attr) (:value attr)}) (:prev-ts attr)))))) -(defn db-before [db ts] +(defn db-before + "How the db was before a given timestamp" + [db ts] (let [indices-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped indices-before :curr-time ts))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 25fd65439..5bd2e9d50 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -2,7 +2,7 @@ (:use core.fdb) [:require [core.manage :as M]]) -(def db-name "hos9") +(def db-name "hos1") (M/reset-db-conn db-name) (def hospital-db (M/get-db-conn db-name)) @@ -40,5 +40,12 @@ (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) +(transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) + + (transact hospital-db (remove-entity :t2-pat1)) + (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) +(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) + @hospital-db +(ind-at @hospital-db 9 :AVET ) From 73428d43f43e3cfb3647563a1fe592318acf886c Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 12 Mar 2014 00:02:27 +0200 Subject: [PATCH 048/102] removing commented-out prints --- functionalDB/src/core/fdb.clj | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 76e85ef3e..56ad108e3 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -225,10 +225,9 @@ (defn- update-indices [indices ent-id attr updated-attr new-val operation] - (let [new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) + (let [ new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation va ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - ; x (println new-eavt) - new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index ] (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) (defn update-datom From 062cdb2a065f170da14f374180b67464472711a6 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 12 Mar 2014 15:46:39 +0200 Subject: [PATCH 049/102] cleanup --- functionalDB/src/core/fdb.clj | 53 +++++++++++++++++------------------ 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 56ad108e3..0e84c567e 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,13 +1,21 @@ (ns core.fdb [:require [clojure.set :as CS :only (union difference )]]) +;; -- EAVT enty-id -> entity {:attrs -> {attr-name -> attr {:value -> the-value}}} +;; -- VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} +;; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) +;; can be used to know who REFs a specific entity +;; -- AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}}} +;; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) +;; can be used to know who are the entities that have a specific attribute with a specific value + (defrecord Database [timestamped top-id curr-time]) (defrecord Indices [EAVT VAET AVET]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) (defn make-db "Create an empty database" [] - (atom (Database. [(Indices. {} {} {})] 0 0))) ;EAVT: all the entity info, vaet for attrs who are of type :db/ref, we hold the back-pointing (from the REFFed entity to the REFing entities) + (atom (Database. [(Indices. {} {} {})] 0 0))) (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) @@ -87,12 +95,12 @@ (defn- update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" - [ent tsVal] + [ent ts-val] (let [ks (keys (:attrs ent)) vls (vals (:attrs ent)) - updatedAttrsVals (map #(assoc % :ts tsVal) vls) - updatedAttrs (zipmap ks updatedAttrsVals) - ](assoc ent :attrs updatedAttrs))) + updated-attrs-vals (map #(assoc % :ts ts-val) vls) + updated-attrs (zipmap ks updated-attrs-vals) + ](assoc ent :attrs updated-attrs))) (defn- remove-entry-from-index [val-to-remove index path ] (let [ old-entries-set (get-in index path )] @@ -105,15 +113,6 @@ remover (partial remove-entry-from-index ent-id)] (reduce remover index paths)))) -; VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} -; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) -; can be used to know who REFs a specific entity - - -;; ; AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}} -;; ; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) -;; ; can be used to know who are the entities that have a specific attribute with a specific value - (defn- add-entry-to-index [index path val-to-add operation] (if (= operation :db/remove) index @@ -158,15 +157,13 @@ (defn remove-entity[db ent-id] (let [ent (entity-at db ent-id) indices (last (:timestamped db)) - vaet (remove-entity-from-index (:VAET indices) ent va ref? ) - ;vaet (update-vaet (:VAET indices) ent disj) - avet (remove-entity-from-index (:AVET indices) ent av indexed?) - ;avet (update-avet (:AVET indices) ent disj) - new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity - new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet) - res (assoc db :timestamped (conj (:timestamped db) new-indices))] - res)) + vaet (remove-entity-from-index (:VAET indices) ent va ref? ) + avet (remove-entity-from-index (:AVET indices) ent av indexed?) + new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity + new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity + new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet) + res (assoc db :timestamped (conj (:timestamped db) new-indices)) + ]res)) (defn transact-on-db [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] @@ -205,9 +202,9 @@ (defn- remove-path-values [old-vals target-val operation] (cond - (= operation :db/add) [] - (= operation :db/reset-to) old-vals - (= operation :db/remove) (collify target-val))) + (= operation :db/add) [] ; nothing to remove + (= operation :db/reset-to) old-vals ; removing all of the old values + (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller (defn- update-index-for-datom [index ent-id attr target-val operation order-fn update-required-pred] @@ -245,8 +242,8 @@ (defn ref-to-as - "returns a seq of all the entities that REFed to a specific entity with the given attr-name (alternativly had an attribute named attr-name whose type is :db/ref - and the value was ent-id), all this at a given time" + "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute + named attr-name whose type is :db/ref and the value was ent-id), all this at a given time" ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] (let [indices ((:timestamped db) ts) From 3cef1f051ea7515e0ff4a3771e5eaa4a6b51fc1e Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 14 Mar 2014 11:12:25 +0200 Subject: [PATCH 050/102] Adding queries towards the ability of asking question of the kind (e?, attr-name, pred-on-value) --- functionalDB/src/core/fdb.clj | 17 ++++++++++++++++- functionalDB/src/scenarios/hospital.clj | 8 ++++++-- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 0e84c567e..0d40cc091 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -128,7 +128,7 @@ (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) relevant-attrs (filter #(filtering-pred %) all-attrs ) - add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:value attr) order-fn (:value attr) :db/add)) + add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) order-fn (:value attr) :db/add)) ] (reduce add-in-index-fn index relevant-attrs))) (defn- remove-entity-from-index[index ent order-fn filtering-pred] @@ -240,6 +240,21 @@ ;;;;;;;;;;;;;;;;;;;;;;; queries +(defn entities-of-ids [db ent-ids] + (let [indices (last (:timestamped db)) + eavt (:EAVT indices)] + (map #(% eavt) ent-ids))) + +(defn filtered-entities-by-attr[db attr-name pred] + (let [indices (last (:timestamped db)) + ve (get-in indices [:AVET :test/machine] ) + relevant-entries (filter #(pred (first %)) ve) + relevant-ids-sets (map second relevant-entries) + relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) + ](entities-of-ids db relevant-ent-ids))) + +(defn entities-by-attr [db attr-name] + (filtered-entities-by-attr [db attr-name #(= % %)])) (defn ref-to-as "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 5bd2e9d50..d891b04d3 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,6 +1,7 @@ (ns scenarios.hospital (:use core.fdb) - [:require [core.manage :as M]]) + [:require [core.manage :as M] + [clojure.set :as CS :only (union difference )]]) (def db-name "hos1") (M/reset-db-conn db-name) @@ -36,7 +37,7 @@ (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) (add-patient :pat1 "London" ["fever" "cough"] ) (add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) -(add-test-results-to-patient :pat1 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) +(add-test-results-to-patient :pat1 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} (fn [a] (= a :test/machine)))) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) @@ -49,3 +50,6 @@ @hospital-db (ind-at @hospital-db 9 :AVET ) + +(filtered-entities-by-attr @hospital-db :test/machine (fn [a] (. a startsWith "X")) ) + From 6b5018431b49df88dfc901ac34e2bfce87511be7 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 14 Mar 2014 22:54:58 +0200 Subject: [PATCH 051/102] adding the evat index and few bug fixes in the remove --- functionalDB/src/core/fdb.clj | 65 +++++++++++++++---------- functionalDB/src/scenarios/hospital.clj | 9 ++-- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 0d40cc091..b53b9d50b 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -5,17 +5,19 @@ ;; -- VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} ;; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) ;; can be used to know who REFs a specific entity -;; -- AVET structed like this: {attrName-> {attrValue -> #{holding-elems-ids}}} +;; -- AVET structed like this: {attrName-> {attr-val -> #{holding-elems-ids}}} ;; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) ;; can be used to know who are the entities that have a specific attribute with a specific value - +;; -- EVAT structed like this: {ent-id -> {attr-val -> #{attr-names}}} +;; this provides the info - for a given ent-id (E) and specificed value (V), what are the attributes that have a specific name (A) at a given time (T). Other way to understand +;; this index is that it answers the question of how a specific entity is related to a specific value. (defrecord Database [timestamped top-id curr-time]) -(defrecord Indices [EAVT VAET AVET]) +(defrecord Indices [EAVT VAET AVET EVAT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) (defn make-db "Create an empty database" [] - (atom (Database. [(Indices. {} {} {})] 0 0))) + (atom (Database. [(Indices. {} {} {} {})] 0 0))) (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) @@ -47,7 +49,8 @@ (defn- ref? [attr] (= :db/ref (:type (meta attr)))) -(defn entity-at ([db ent-id ts] (ent-id ((:timestamped db) ts))) +(defn entity-at "the entity with the given ent-id at the given time (defualts to the latest time)" + ([db ent-id ts] (get-in db [:timestamped ts :EAVT ent-id])) ([db ent-id] (entity-at db ent-id (:curr-time db))) ) (defn attr-at @@ -77,8 +80,9 @@ (let [attr-id (keyword (:name attr))] (assoc-in ent [:attrs attr-id] attr))) -(defn- av [attr vl] [attr vl] ) -(defn- va [attr vl] [vl attr] ) +(defn- ave [ent-id attr vl] [attr vl ent-id] ) +(defn- vae [ent-id attr vl] [vl attr ent-id] ) +(defn- eva [ent-id attr vl] [ent-id vl attr] ) (defn- next-ts [db] "returns the next timestamp of the given database" @@ -102,26 +106,32 @@ updated-attrs (zipmap ks updated-attrs-vals) ](assoc ent :attrs updated-attrs))) -(defn- remove-entry-from-index [val-to-remove index path ] - (let [ old-entries-set (get-in index path )] - (assoc-in index path (disj old-entries-set val-to-remove)))) +(defn- remove-entry-from-index [index path ] + (let [ stam (println path) + remove-path (butlast path) + val-to-remove (last path) + old-entries-set (get-in index remove-path )] + (assoc-in index remove-path (disj old-entries-set val-to-remove)))) (defn- remove-entries-from-index [index attr-name ent-id order-fn path-values operation] (if (= operation :db/add) index - (let [paths (map #(order-fn attr-name %) path-values) - remover (partial remove-entry-from-index ent-id)] - (reduce remover index paths)))) + (let [paths (map #(order-fn ent-id attr-name %) path-values) + ; remover (partial remove-entry-from-index ent-id) + ] + (reduce remove-entry-from-index index paths)))) -(defn- add-entry-to-index [index path val-to-add operation] +(defn- add-entry-to-index [index path operation] (if (= operation :db/remove) index - (let [to-be-updated-set (get-in index path #{})] - (assoc-in index path (conj to-be-updated-set val-to-add))))) + (let [update-path (butlast path) + update-value (last path) + to-be-updated-set (get-in index update-path #{})] + (assoc-in index update-path (conj to-be-updated-set update-value) )))) (defn- update-attr-in-index [index ent-id attr-name order-fn target-val operation] (let [ colled-target-val (collify target-val) - add-entry-fn (fn[indx vl] (add-entry-to-index indx (order-fn attr-name vl) ent-id operation)) + add-entry-fn (fn[indx vl] (add-entry-to-index indx (order-fn ent-id attr-name vl) operation)) ] (reduce add-entry-fn index colled-target-val))) (defn- add-entity-to-index [index ent order-fn filtering-pred] @@ -135,7 +145,7 @@ (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) relevant-attrs (filter #(filtering-pred %) all-attrs ) - remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:value attr) ent-id order-fn (collify (:value attr)) :db/remove)) + remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:name attr) ent-id order-fn (collify (:value attr)) :db/remove)) ] (reduce remove-from-index-fn index relevant-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one @@ -145,9 +155,10 @@ indices (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) new-eavt (assoc (:EAVT indices) ent-id (update-creation-ts fixed-ent new-ts) ) - new-vaet (add-entity-to-index (:VAET indices) ent va ref?) - new-avet (add-entity-to-index (:AVET indices) ent av indexed?) - new-indices (assoc indices :VAET new-vaet :EAVT new-eavt :AVET new-avet ) + new-vaet (add-entity-to-index (:VAET indices) ent vae ref?) + new-avet (add-entity-to-index (:AVET indices) ent ave indexed?) + new-evat (add-entity-to-index (:EVAT indices) ent eva indexed?) + new-indices (assoc indices :VAET new-vaet :EAVT new-eavt :AVET new-avet :EVAT new-evat) ](assoc db :timestamped (conj (:timestamped db) new-indices) :top-id next-top))) @@ -157,11 +168,12 @@ (defn remove-entity[db ent-id] (let [ent (entity-at db ent-id) indices (last (:timestamped db)) - vaet (remove-entity-from-index (:VAET indices) ent va ref? ) - avet (remove-entity-from-index (:AVET indices) ent av indexed?) + vaet (remove-entity-from-index (:VAET indices) ent vae ref? ) + avet (remove-entity-from-index (:AVET indices) ent ave indexed?) + evat (remove-entity-from-index (:EVAT indices) ent eva indexed?) new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet) + new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet :EVAT evat) res (assoc db :timestamped (conj (:timestamped db) new-indices)) ]res)) @@ -223,8 +235,9 @@ (defn- update-indices [indices ent-id attr updated-attr new-val operation] (let [ new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) - new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation va ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation av indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation vae ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation ave indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + new-evat (update-index-for-datom (:EVAT indices) ent-id attr new-val operation eva indexed?) ] (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) (defn update-datom diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index d891b04d3..24b428bc4 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -3,7 +3,7 @@ [:require [core.manage :as M] [clojure.set :as CS :only (union difference )]]) -(def db-name "hos1") +(def db-name "hos12") (M/reset-db-conn db-name) (def hospital-db (M/get-db-conn db-name)) @@ -31,11 +31,13 @@ (defn add-test-results-to-patient [pat-id test-result] (let [test-id (:id test-result) ] (transact hospital-db (add-entity test-result)) - (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) + (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)) + )) ;; world setup (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) (add-patient :pat1 "London" ["fever" "cough"] ) +@hospital-db (add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) (add-test-results-to-patient :pat1 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} (fn [a] (= a :test/machine)))) @@ -48,8 +50,9 @@ (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) + @hospital-db -(ind-at @hospital-db 9 :AVET ) +(ind-at @hospital-db 9 :EVAT) (filtered-entities-by-attr @hospital-db :test/machine (fn [a] (. a startsWith "X")) ) From 735ca11e73d87f6723b3f47aa1ba101ed4138c71 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 14 Mar 2014 23:01:15 +0200 Subject: [PATCH 052/102] remove prints --- functionalDB/src/core/fdb.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index b53b9d50b..60fe2314e 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -107,8 +107,7 @@ ](assoc ent :attrs updated-attrs))) (defn- remove-entry-from-index [index path ] - (let [ stam (println path) - remove-path (butlast path) + (let [ remove-path (butlast path) val-to-remove (last path) old-entries-set (get-in index remove-path )] (assoc-in index remove-path (disj old-entries-set val-to-remove)))) @@ -258,10 +257,10 @@ eavt (:EAVT indices)] (map #(% eavt) ent-ids))) -(defn filtered-entities-by-attr[db attr-name pred] +(defn filtered-entities-by-attr[db attr-name val-pred] (let [indices (last (:timestamped db)) ve (get-in indices [:AVET :test/machine] ) - relevant-entries (filter #(pred (first %)) ve) + relevant-entries (filter #(val-pred (first %)) ve) relevant-ids-sets (map second relevant-entries) relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) ](entities-of-ids db relevant-ent-ids))) @@ -293,6 +292,6 @@ (assoc db :timestamped indices-before :curr-time ts))) (defn ind-at - "inspecting a specific index at a given time. The kind argument may be of of these: :AVET :EAVT :VAET " + "inspecting a specific index at a given time. The kind argument may be of of these: :AVET :EAVT :VAET :EVAT" [db ts kind] (kind ((:timestamped db) ts))) From bbeca1050cb3a071207948d7a61ac52d4c1bfdb4 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 17 Mar 2014 12:08:05 +0200 Subject: [PATCH 053/102] fixing indentation, asserting using preconditions the operation of datom updates and attr cardinality / indexing upon creation --- functionalDB/src/core/fdb.clj | 68 ++++++++++++++----------- functionalDB/src/scenarios/hospital.clj | 2 +- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 60fe2314e..f0128f681 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,5 +1,5 @@ (ns core.fdb - [:require [clojure.set :as CS :only (union difference )]]) + [:require [clojure.set :as CS :only (union difference intersection)]]) ;; -- EAVT enty-id -> entity {:attrs -> {attr-name -> attr {:value -> the-value}}} ;; -- VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} @@ -36,6 +36,7 @@ :db/remove - removes the given set of values from the attribute's current set of values" ([name value type ; these ones are required & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] + {:pre [(contains? #{true false} indexed) (contains? #{:db/single :db/multiple} cardinality)]} (with-meta (Attr. name value -1 -1) {:type type :indexed indexed :cardinality cardinality} ))) (defn collify [x] (if (coll? x) x [x])) @@ -51,7 +52,7 @@ (defn entity-at "the entity with the given ent-id at the given time (defualts to the latest time)" ([db ent-id ts] (get-in db [:timestamped ts :EAVT ent-id])) - ([db ent-id] (entity-at db ent-id (:curr-time db))) ) + ([db ent-id] (entity-at db ent-id (:curr-time db))) ) (defn attr-at "The attribute of an entity at a given time (defaults to recent time)" @@ -94,30 +95,30 @@ (let [top-id (:top-id db) ent-id (:id ent) inceased-id (inc top-id) - [id-to-use next-top] (if (= ent-id :db/no-id-yet) [(keyword (str inceased-id)) inceased-id] [ent-id top-id])] - [id-to-use next-top])) + [id-to-use next-top] (if (= ent-id :db/no-id-yet) + [(keyword (str inceased-id)) inceased-id] + [ent-id top-id])] + [id-to-use next-top])) (defn- update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent ts-val] (let [ks (keys (:attrs ent)) - vls (vals (:attrs ent)) - updated-attrs-vals (map #(assoc % :ts ts-val) vls) - updated-attrs (zipmap ks updated-attrs-vals) + vls (vals (:attrs ent)) + updated-attrs-vals (map #(assoc % :ts ts-val) vls) + updated-attrs (zipmap ks updated-attrs-vals) ](assoc ent :attrs updated-attrs))) (defn- remove-entry-from-index [index path ] - (let [ remove-path (butlast path) - val-to-remove (last path) - old-entries-set (get-in index remove-path )] + (let [remove-path (butlast path) + val-to-remove (last path) + old-entries-set (get-in index remove-path )] (assoc-in index remove-path (disj old-entries-set val-to-remove)))) (defn- remove-entries-from-index [index attr-name ent-id order-fn path-values operation] (if (= operation :db/add) index - (let [paths (map #(order-fn ent-id attr-name %) path-values) - ; remover (partial remove-entry-from-index ent-id) - ] + (let [paths (map #(order-fn ent-id attr-name %) path-values)] (reduce remove-entry-from-index index paths)))) (defn- add-entry-to-index [index path operation] @@ -178,15 +179,17 @@ (defn transact-on-db [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] - (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) - (let [ initial-indices (:timestamped initial-db) - new-indices (last (:timestamped transacted)) - res (assoc initial-db :timestamped (conj initial-indices new-indices) - :curr-time (next-ts initial-db) - :top-id (:top-id transacted))] + (if tx + (recur rst-tx (apply (first tx) transacted (rest tx))) + (let [initial-indices (:timestamped initial-db) + new-indices (last (:timestamped transacted)) + res (assoc initial-db + :timestamped (conj initial-indices new-indices) + :curr-time (next-ts initial-db) + :top-id (:top-id transacted))] res)))) -(defmacro _transact [db op & txs] +(defmacro _transact [db op & txs] (when txs (loop [[frst-tx# & rst-tx#] txs res# [op db 'transact-on-db] accum-txs# []] (if frst-tx# @@ -207,7 +210,10 @@ (assoc attr :ts new-ts :prev-ts ( :ts attr))) (defn- update-attr [attr new-val new-ts operation] - (-> attr + {:pre [ (if (single? attr) + (= operation :db/reset-to) + (contains? #{:db/reset-to :db/add :db/remove} operation))]} + (-> attr (update-attr-modification-time new-ts) (update-attr-value new-val operation))) @@ -232,10 +238,10 @@ (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) (defn- update-indices [indices ent-id attr updated-attr new-val operation] - (let [ new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) - new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation vae ref?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index - new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation ave indexed?) ; intentionally attr is passed and not updated-attr, we need to use the old value to locate where to update in the index + ; in the next indices updates the attr is intentionally passed and not updated-attr, we need to use the old value to locate where to update in the index + new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation vae ref?) + new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation ave indexed?) new-evat (update-index-for-datom (:EVAT indices) ent-id attr new-val operation eva indexed?) ] (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) @@ -252,26 +258,28 @@ ;;;;;;;;;;;;;;;;;;;;;;; queries -(defn entities-of-ids [db ent-ids] +(defn entities-of-ids + "for a given seq of entity ids, return the real entities" + [db ent-ids] (let [indices (last (:timestamped db)) eavt (:EAVT indices)] (map #(% eavt) ent-ids))) -(defn filtered-entities-by-attr[db attr-name val-pred] +(defn entities-by-AV [db attr-name val-pred] (let [indices (last (:timestamped db)) ve (get-in indices [:AVET :test/machine] ) relevant-entries (filter #(val-pred (first %)) ve) relevant-ids-sets (map second relevant-entries) relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) - ](entities-of-ids db relevant-ent-ids))) + ](entities-of-ids db relevant-ent-ids))) -(defn entities-by-attr [db attr-name] - (filtered-entities-by-attr [db attr-name #(= % %)])) +(defn entities-by-A [db attr-name] + (entities-by-AV [db attr-name #(= % %)])) (defn ref-to-as "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute named attr-name whose type is :db/ref and the value was ent-id), all this at a given time" - ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] (let [indices ((:timestamped db) ts) reffing-ids (get-in indices [:VAET ent-id attr-name])] diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 24b428bc4..7835d4527 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -54,5 +54,5 @@ @hospital-db (ind-at @hospital-db 9 :EVAT) -(filtered-entities-by-attr @hospital-db :test/machine (fn [a] (. a startsWith "X")) ) +(entities-by-AV @hospital-db :test/machine (fn [a] (. a startsWith "X")) ) From 8dd4762a6e35ec02fc61b0722480e9ebf04cdf0a Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 17 Mar 2014 12:16:08 +0200 Subject: [PATCH 054/102] calling with the right set of args --- functionalDB/src/core/fdb.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index f0128f681..94f4a368f 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -274,7 +274,7 @@ ](entities-of-ids db relevant-ent-ids))) (defn entities-by-A [db attr-name] - (entities-by-AV [db attr-name #(= % %)])) + (entities-by-AV db attr-name #(= % %))) (defn ref-to-as "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute From 329c4b0e8b17e74c68a72461712fe640ac15812e Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 17 Mar 2014 12:16:54 +0200 Subject: [PATCH 055/102] indentation --- functionalDB/src/core/fdb.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 94f4a368f..1f8a71797 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -210,7 +210,7 @@ (assoc attr :ts new-ts :prev-ts ( :ts attr))) (defn- update-attr [attr new-val new-ts operation] - {:pre [ (if (single? attr) + {:pre [(if (single? attr) (= operation :db/reset-to) (contains? #{:db/reset-to :db/add :db/remove} operation))]} (-> attr From bdfd6ba84563426b531ed9626885f94fc49d5e25 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 21 Mar 2014 10:28:50 +0200 Subject: [PATCH 056/102] adding the ability to query the database using simple, map datalog queries. --- functionalDB/src/core/fdb.clj | 182 +++++++++++++++++++----- functionalDB/src/scenarios/hospital.clj | 22 ++- 2 files changed, 159 insertions(+), 45 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 1f8a71797..508b9c007 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -81,13 +81,13 @@ (let [attr-id (keyword (:name attr))] (assoc-in ent [:attrs attr-id] attr))) -(defn- ave [ent-id attr vl] [attr vl ent-id] ) -(defn- vae [ent-id attr vl] [vl attr ent-id] ) -(defn- eva [ent-id attr vl] [ent-id vl attr] ) +(defn ave [e a v] [a v e] ) +(defn vae [e a v] [v a e] ) +(defn eva [e a v] [e v a] ) -(defn- next-ts [db] - "returns the next timestamp of the given database" - (inc (:curr-time db))) +(defn ave-to-eav [a v e] [e a v]) + +(defn- next-ts [db] (inc (:curr-time db))) (defn- next-id "returns a pair composed of the id to use for the given entity and the next free running id in the database" @@ -109,39 +109,45 @@ updated-attrs (zipmap ks updated-attrs-vals) ](assoc ent :attrs updated-attrs))) -(defn- remove-entry-from-index [index path ] +(defn- remove-entry-from-index + [index path] (let [remove-path (butlast path) val-to-remove (last path) old-entries-set (get-in index remove-path )] (assoc-in index remove-path (disj old-entries-set val-to-remove)))) -(defn- remove-entries-from-index [index attr-name ent-id order-fn path-values operation] +(defn- remove-entries-from-index + [index attr-name ent-id order-fn path-values operation] (if (= operation :db/add) index (let [paths (map #(order-fn ent-id attr-name %) path-values)] (reduce remove-entry-from-index index paths)))) -(defn- add-entry-to-index [index path operation] +(defn- add-entry-to-index + [index path operation] (if (= operation :db/remove) index (let [update-path (butlast path) update-value (last path) - to-be-updated-set (get-in index update-path #{})] + to-be-updated-set (get-in index update-path #{})] (assoc-in index update-path (conj to-be-updated-set update-value) )))) -(defn- update-attr-in-index [index ent-id attr-name order-fn target-val operation] +(defn- update-attr-in-index + [index ent-id attr-name order-fn target-val operation] (let [ colled-target-val (collify target-val) - add-entry-fn (fn[indx vl] (add-entry-to-index indx (order-fn ent-id attr-name vl) operation)) + add-entry-fn (fn [indx vl] (add-entry-to-index indx (order-fn ent-id attr-name vl) operation)) ] (reduce add-entry-fn index colled-target-val))) -(defn- add-entity-to-index [index ent order-fn filtering-pred] +(defn- add-entity-to-index + [index ent order-fn filtering-pred] (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) relevant-attrs (filter #(filtering-pred %) all-attrs ) add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) order-fn (:value attr) :db/add)) ] (reduce add-in-index-fn index relevant-attrs))) -(defn- remove-entity-from-index[index ent order-fn filtering-pred] +(defn- remove-entity-from-index + [index ent order-fn filtering-pred] (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) relevant-attrs (filter #(filtering-pred %) all-attrs ) @@ -149,7 +155,8 @@ ] (reduce remove-from-index-fn index relevant-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity [db ent] +(defn add-entity + [db ent] (let [[ent-id next-top] (next-id db ent) new-ts (next-ts db) indices (last (:timestamped db)) @@ -162,10 +169,12 @@ ](assoc db :timestamped (conj (:timestamped db) new-indices) :top-id next-top))) -(defn add-entities [db ents-seq] +(defn add-entities + [db ents-seq] (reduce add-entity db ents-seq)) -(defn remove-entity[db ent-id] +(defn remove-entity + [db ent-id] (let [ent (entity-at db ent-id) indices (last (:timestamped db)) vaet (remove-entity-from-index (:VAET indices) ent vae ref? ) @@ -177,7 +186,8 @@ res (assoc db :timestamped (conj (:timestamped db) new-indices)) ]res)) -(defn transact-on-db [initial-db txs] +(defn transact-on-db + [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) @@ -189,27 +199,34 @@ :top-id (:top-id transacted))] res)))) -(defmacro _transact [db op & txs] +(defmacro _transact + [db op & txs] (when txs (loop [[frst-tx# & rst-tx#] txs res# [op db 'transact-on-db] accum-txs# []] (if frst-tx# (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) (list* (conj res# accum-txs#)))))) -(defn _what-if [ db f txs] (f db txs)) +(defn _what-if + [ db f txs] + (f db txs)) -(defmacro what-if [db & txs] +(defmacro what-if + [db & txs] `(_transact ~db _what-if ~@txs)) -(defmacro transact [db & txs] +(defmacro transact + [db & txs] `(_transact ~db swap! ~@txs)) ;;;;;;;;;;;;;;;;;;;;;;; datom updates -(defn- update-attr-modification-time [attr new-ts] +(defn- update-attr-modification-time + [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) -(defn- update-attr [attr new-val new-ts operation] +(defn- update-attr + [attr new-val new-ts operation] {:pre [(if (single? attr) (= operation :db/reset-to) (contains? #{:db/reset-to :db/add :db/remove} operation))]} @@ -217,7 +234,8 @@ (update-attr-modification-time new-ts) (update-attr-value new-val operation))) -(defn- remove-path-values [old-vals target-val operation] +(defn- remove-path-values + [old-vals target-val operation] (cond (= operation :db/add) [] ; nothing to remove (= operation :db/reset-to) old-vals ; removing all of the old values @@ -234,7 +252,8 @@ updated-index (update-attr-in-index cleaned-index ent-id attr-name order-fn target-val operation) ] updated-index))) -(defn- update-eavt-for-datom [eavt ent-id new-attr] +(defn- update-eavt-for-datom + [eavt ent-id new-attr] (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) (defn- update-indices [indices ent-id attr updated-attr new-val operation] @@ -246,7 +265,8 @@ ] (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) (defn update-datom - ([db ent-id att-name new-val] (update-datom db ent-id att-name new-val :db/reset-to )) + ([db ent-id att-name new-val] + (update-datom db ent-id att-name new-val :db/reset-to )) ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) (let [update-ts (next-ts db) indices (last (:timestamped db)) @@ -258,6 +278,100 @@ ;;;;;;;;;;;;;;;;;;;;;;; queries +(defn variable? + "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" + [x] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF + (or (= x "_") (= (first x) \?))) + +(defmacro clause-item-meta + "Finds the name of the variable at an element of a datalog clause item" + [clause-item] + (if (coll? clause-item) + (first (filter #(variable? %) (map #(str %) clause-item))) ; need to call first to that the only element in the collection + (str clause-item))) + +(defmacro clause-item-expr + "Create a predicate for each element in the datalog clause" + [clause-item] + (cond + (variable? (str clause-item)) #(= % %) ; simple one, was something like ?a + (not (coll? clause-item)) `#(= % ~(identity clause-item)) ; simple value given, was something like :likes + (= 2 (count clause-item)) `#(~(first clause-item) %) ; was something like (pos? ?a) + (variable? (str (second clause-item))) `#(~(first clause-item) % ~(last clause-item)) ; was something like (> ?a 42) + (variable? (str (last clause-item))) `#(~(first clause-item) ~(second clause-item) %))) ; was something like (> 42 ?a) + +(defmacro q-clause [clause] + "The aim of this macro is to return for a datalog clause (a vector with three elements describing EAV) a vector of predicates to operate on + an index containing EAV, and set for that vector its metadata to be the names of the variables that the user assiged for each item in the triplet" + (loop [[frst# & rst#] clause exprs# [] metas# [] ] + (if frst# + (recur rst# (conj exprs# `(clause-item-expr ~ frst#)) (conj metas#`(clause-item-meta ~ frst#))) + (with-meta exprs# {:db/variable metas#})))) + + (defmacro q-clauses [clauses] + (loop [[frst# & rst#] clauses preds-vecs# [] ] + (if-not frst# preds-vecs# + (recur rst# `(conj ~preds-vecs# (q-clause ~frst#))) ))) + + (defn choose-index ;; TODO !!! at the moment not optimzing, just returning always the AVET index + "Upon receiving a database and query clauses, this function responsible to deduce on which index in the db it is best to perfom the query clauses, and then return + a vector in which the first element is the decided index and the second element is a function that knows how to restore an EAV structure from that decided index path structure." + [db query] + [(ind-at db :AVET) ave ave-to-eav]) + +(defn filter-index + "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: + -- At index 0 : the elements (e-ids or attribute names) that are found at the leave of the path. + -- At index 1 : the key used at the first level of the path. + -- At index 2 : the key used at the second level of the path" ; the triplet structure results in a EAV structure for the common usage + [index path-preds from-eav to-eav] + (for [ path-pred path-preds + :let [[lvl1-prd lvl2-prd lvl3-prd] (apply from-eav path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta + [k1 l2map] index ; keys and values of the first level + :when (lvl1-prd k1) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate + [k2 l3-set] l2map ; keys and values of the second level + :when (lvl2-prd k2) ; filtering to keep only the keys and vals of keys that passed the second level predicate + :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them + (with-meta (to-eav k1 k2 res) (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables + + (defn items-that-answer-all-conditions + "takes the sequence of all the items collection, each such collection answered one condition, we test here what are the items that answered all of the conditions + i.e., what items are found at exactly 'num-of-conditions' of such collections " + [items-seq num-of-conditions] + (->> items-seq ; take the items-seq + (map vec) ; make each collection (actually a set) into a vector + (reduce into []) ;reduce all the vectors into one big vector + (frequencies) ; count for each item in how many collections (sets) it was in originally + (filter #(= num-of-conditions (last %))) ; keep only the items that answered all of the conditions + (map first) ; take from the duos the items themeselves + (set))) ; return it as set + +(defn mask-path-leaf-with-items + "a path is a vector triplet, at the first position there's a set of the path's items, here we intersect the path's items with the set of relevant items" + [path relevant-items] + (assoc path 0 (CS/intersection relevant-items (path 0)))) + +(defn query-index + "an index is a 3 level map (VAE, AVE, EVA) that is found at a specific time (T). + The path-preds argument is a seq of vectors. Each of these vectors contains two elements: + -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, + E -> an id matching predicator) + -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. + -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] + The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" + [index path-preds from-eav to-eav] + (let [filtered-paths (filter-index index path-preds from-eav to-eav) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path + relevant-elements (items-that-answer-all-conditions (map first filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths + relevant-paths (map mask-path-leaf-with-items filtered-paths (repeat relevant-elements))] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates + (filter #(not-empty (% 0)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items + +(defmacro q + "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]})" + [db query] + `(let [query# (q-clauses ~(:where query) ) + [ind# from-eav# to-eav#] (choose-index ~db query#)] + (query-index ind# query# from-eav# to-eav#))) + (defn entities-of-ids "for a given seq of entity ids, return the real entities" [db ent-ids] @@ -265,7 +379,8 @@ eavt (:EAVT indices)] (map #(% eavt) ent-ids))) -(defn entities-by-AV [db attr-name val-pred] +(defn entities-by-AV + [db attr-name val-pred] (let [indices (last (:timestamped db)) ve (get-in indices [:AVET :test/machine] ) relevant-entries (filter #(val-pred (first %)) ve) @@ -273,7 +388,8 @@ relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) ](entities-of-ids db relevant-ent-ids))) -(defn entities-by-A [db attr-name] +(defn entities-by-A + [db attr-name] (entities-by-AV db attr-name #(= % %))) (defn ref-to-as @@ -300,6 +416,8 @@ (assoc db :timestamped indices-before :curr-time ts))) (defn ind-at - "inspecting a specific index at a given time. The kind argument may be of of these: :AVET :EAVT :VAET :EVAT" - [db ts kind] - (kind ((:timestamped db) ts))) + "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :EAVT :VAET :EVAT" + ([db kind] + (ind-at db kind (:curr-time db))) + ([db kind ts] + (kind ((:timestamped db) ts)))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 7835d4527..92c588a41 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -31,28 +31,24 @@ (defn add-test-results-to-patient [pat-id test-result] (let [test-id (:id test-result) ] (transact hospital-db (add-entity test-result)) - (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)) - )) + (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) ;; world setup (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) (add-patient :pat1 "London" ["fever" "cough"] ) +(add-patient :pat2 "London" ["fever" "cough"] ) @hospital-db -(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] (= a :test/machine)))) -(add-test-results-to-patient :pat1 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} (fn [a] (= a :test/machine)))) +(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] true))) +(add-test-results-to-patient :pat2 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} (fn [a] true))) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) -(transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) +;(transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) - (transact hospital-db (remove-entity :t2-pat1)) + ;(transact hospital-db (remove-entity :t2-pat1)) -(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) -(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) +(q @hospital-db {:where [[ ?e :test/bp-systolic (> 150 ?b)] [ ?e :test/bp-diastolic ?k]]} ) - -@hospital-db -(ind-at @hospital-db 9 :EVAT) - -(entities-by-AV @hospital-db :test/machine (fn [a] (. a startsWith "X")) ) +;; (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) +;; (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) From a3dfd74fe89e5c11a571386123a923eb80a3b14d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 21 Mar 2014 12:54:02 +0200 Subject: [PATCH 057/102] updating the meta-query of a non-vriable to be :db/no-var instead of just a string of the item --- functionalDB/src/core/fdb.clj | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 508b9c007..b6d4168c0 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -283,12 +283,20 @@ [x] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF (or (= x "_") (= (first x) \?))) +(defn ind-at + "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :EAVT :VAET :EVAT" + ([db kind] + (ind-at db kind (:curr-time db))) + ([db kind ts] + (kind ((:timestamped db) ts)))) + (defmacro clause-item-meta - "Finds the name of the variable at an element of a datalog clause item" + "Finds the name of the variable at an item of a datalog clause element. If no variable, returning :db/no-var" [clause-item] - (if (coll? clause-item) - (first (filter #(variable? %) (map #(str %) clause-item))) ; need to call first to that the only element in the collection - (str clause-item))) + (cond + (coll? clause-item) (first (filter #(variable? %) (map #(str %) clause-item))) ; the item is an s-expression, need to treat it as a coll, by going over it and returning the name of the variable + (variable? (str clause-item)) (str clause-item) ; the item is a simple variable + :no-variable-in-clause :db/no-var)) ; the item is a value and not a variable (defmacro clause-item-expr "Create a predicate for each element in the datalog clause" @@ -414,10 +422,3 @@ [db ts] (let [indices-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped indices-before :curr-time ts))) - -(defn ind-at - "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :EAVT :VAET :EVAT" - ([db kind] - (ind-at db kind (:curr-time db))) - ([db kind ts] - (kind ((:timestamped db) ts)))) From 6ef227b7c7f6f04be3946a94d879e21439d554b2 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 22 Mar 2014 12:57:06 +0200 Subject: [PATCH 058/102] factoring out storage (instead of treating it as a covering index) --- functionalDB/src/core/fdb.clj | 168 +++++++++++++++--------------- functionalDB/src/core/storage.clj | 10 ++ 2 files changed, 96 insertions(+), 82 deletions(-) create mode 100644 functionalDB/src/core/storage.clj diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index b6d4168c0..93983dfe7 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,7 +1,8 @@ (ns core.fdb + [:use core.storage] [:require [clojure.set :as CS :only (union difference intersection)]]) -;; -- EAVT enty-id -> entity {:attrs -> {attr-name -> attr {:value -> the-value}}} +;; -- storage enty-id -> entity {:attrs -> {attr-name -> attr {:value -> the-value}}} ;; -- VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} ;; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) ;; can be used to know who REFs a specific entity @@ -12,12 +13,12 @@ ;; this provides the info - for a given ent-id (E) and specificed value (V), what are the attributes that have a specific name (A) at a given time (T). Other way to understand ;; this index is that it answers the question of how a specific entity is related to a specific value. (defrecord Database [timestamped top-id curr-time]) -(defrecord Indices [EAVT VAET AVET EVAT]) +(defrecord Timestamped [storage VAET AVET EVAT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) (defn make-db "Create an empty database" [] - (atom (Database. [(Indices. {} {} {} {})] 0 0))) + (atom (Database. [(Timestamped. (initial-storage) {} {} {})] 0 0))) (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) @@ -50,15 +51,17 @@ (defn- ref? [attr] (= :db/ref (:type (meta attr)))) -(defn entity-at "the entity with the given ent-id at the given time (defualts to the latest time)" - ([db ent-id ts] (get-in db [:timestamped ts :EAVT ent-id])) - ([db ent-id] (entity-at db ent-id (:curr-time db))) ) +(defn entity-at + "the entity with the given ent-id at the given time (defualts to the latest time)" + ([db ent-id] (entity-at db ent-id (:curr-time db))) + ([db ent-id ts] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) (defn attr-at "The attribute of an entity at a given time (defaults to recent time)" - ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name] + (attr-at db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] - (let [indices ((:timestamped db) ts)] (get-in indices [:EAVT ent-id :attrs attr-name])))) + (get-in (entity-at db ent-id ts) [:attrs attr-name]))) (defn value-of-at "value of a datom at a given time, if no time is provided, we default to the most recent value" @@ -159,14 +162,14 @@ [db ent] (let [[ent-id next-top] (next-id db ent) new-ts (next-ts db) - indices (last (:timestamped db)) + timestamped (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) - new-eavt (assoc (:EAVT indices) ent-id (update-creation-ts fixed-ent new-ts) ) - new-vaet (add-entity-to-index (:VAET indices) ent vae ref?) - new-avet (add-entity-to-index (:AVET indices) ent ave indexed?) - new-evat (add-entity-to-index (:EVAT indices) ent eva indexed?) - new-indices (assoc indices :VAET new-vaet :EAVT new-eavt :AVET new-avet :EVAT new-evat) - ](assoc db :timestamped (conj (:timestamped db) new-indices) + new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) );(assoc (:storage timestamped) ent-id (update-creation-ts fixed-ent new-ts) ) + new-vaet (add-entity-to-index (:VAET timestamped) ent vae ref?) + new-avet (add-entity-to-index (:AVET timestamped) ent ave indexed?) + new-evat (add-entity-to-index (:EVAT timestamped) ent eva indexed?) + new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EVAT new-evat) + ](assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top))) (defn add-entities @@ -176,14 +179,14 @@ (defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) - indices (last (:timestamped db)) - vaet (remove-entity-from-index (:VAET indices) ent vae ref? ) - avet (remove-entity-from-index (:AVET indices) ent ave indexed?) - evat (remove-entity-from-index (:EVAT indices) ent eva indexed?) - new-eavt (dissoc (:EAVT indices) ent-id) ; removing the entity + timestamped (last (:timestamped db)) + vaet (remove-entity-from-index (:VAET timestamped) ent vae ref? ) + avet (remove-entity-from-index (:AVET timestamped) ent ave indexed?) + evat (remove-entity-from-index (:EVAT timestamped) ent eva indexed?) + new-storage (remove-entity-from-storage (:storage timestamped) ent) ;(dissoc (:storage timestamped) ent-id) ; removing the entity new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity - new-indices (assoc indices :EAVT new-eavt :VAET new-vaet :AVET avet :EVAT evat) - res (assoc db :timestamped (conj (:timestamped db) new-indices)) + new-timestamped (assoc timestamped :storage new-storage :VAET new-vaet :AVET avet :EVAT evat) + res (assoc db :timestamped (conj (:timestamped db) new-timestamped)) ]res)) (defn transact-on-db @@ -191,10 +194,10 @@ (loop [[tx & rst-tx] txs transacted initial-db] (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) - (let [initial-indices (:timestamped initial-db) - new-indices (last (:timestamped transacted)) + (let [initial-timestamped (:timestamped initial-db) + new-timestamped (last (:timestamped transacted)) res (assoc initial-db - :timestamped (conj initial-indices new-indices) + :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted))] res)))) @@ -241,40 +244,40 @@ (= operation :db/reset-to) old-vals ; removing all of the old values (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller -(defn- update-index-for-datom - [index ent-id attr target-val operation order-fn update-required-pred] - (if-not (update-required-pred attr) +(defn- update-index + [index ent-id old-attr target-val operation order-fn update-required-pred] + (if-not (update-required-pred old-attr) index - (let [old-vals (:value attr) - attr-name (:name attr) + (let [old-vals (:value old-attr) + attr-name (:name old-attr) remove-paths (remove-path-values old-vals target-val operation) cleaned-index (remove-entries-from-index index attr-name ent-id order-fn remove-paths operation) - updated-index (update-attr-in-index cleaned-index ent-id attr-name order-fn target-val operation) - ] updated-index))) + updated-index (update-attr-in-index cleaned-index ent-id attr-name order-fn target-val operation)] + updated-index))) -(defn- update-eavt-for-datom - [eavt ent-id new-attr] - (assoc-in eavt [ent-id :attrs (:name new-attr)] new-attr)) +(defn update-entity [storage e-id new-attr] + (assoc-in (stored-entity storage ent-id) [:attrs (:name new-attr)])) -(defn- update-indices [indices ent-id attr updated-attr new-val operation] - (let [ new-eavt (update-eavt-for-datom (:EAVT indices) ent-id updated-attr) - ; in the next indices updates the attr is intentionally passed and not updated-attr, we need to use the old value to locate where to update in the index - new-vaet (update-index-for-datom (:VAET indices) ent-id attr new-val operation vae ref?) - new-avet (update-index-for-datom (:AVET indices) ent-id attr new-val operation ave indexed?) - new-evat (update-index-for-datom (:EVAT indices) ent-id attr new-val operation eva indexed?) - ] (assoc indices :EAVT new-eavt :VAET new-vaet :AVET new-avet))) +(defn- update-timestamped + [timestamped ent-id old-attr updated-attr new-val operation] + (let [ storage (:storage timestamped) + new-storage (update-storage storage (update-entity storage ent-id updated-attr)) + new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation vae ref?) + new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation ave indexed?) + new-evat (update-index (:EVAT timestamped) ent-id old-attr new-val operation eva indexed?)] + (assoc timestamped :storage new-storage :VAET new-vaet :AVET new-avet :EVAT new-evat))) (defn update-datom - ([db ent-id att-name new-val] - (update-datom db ent-id att-name new-val :db/reset-to )) - ([db ent-id att-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) + ([db ent-id attr-name new-val] + (update-datom db ent-id attr-name new-val :db/reset-to )) + ([db ent-id attr-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) (let [update-ts (next-ts db) - indices (last (:timestamped db)) - attr (get-in indices [:EAVT ent-id :attrs att-name]) + timestamped (last (:timestamped db)) + attr (attr-at db ent-id attr-name) updated-attr (update-attr attr new-val update-ts operation) - fully-updated-indices (update-indices indices ent-id attr updated-attr new-val operation) - new-db (update-in db [:timestamped] conj fully-updated-indices) - ]new-db))) + fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation) + new-db (update-in db [:timestamped] conj fully-updated-timestamped)] + new-db))) ;;;;;;;;;;;;;;;;;;;;;;; queries @@ -284,7 +287,7 @@ (or (= x "_") (= (first x) \?))) (defn ind-at - "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :EAVT :VAET :EVAT" + "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EVAT" ([db kind] (ind-at db kind (:curr-time db))) ([db kind ts] @@ -380,35 +383,6 @@ [ind# from-eav# to-eav#] (choose-index ~db query#)] (query-index ind# query# from-eav# to-eav#))) -(defn entities-of-ids - "for a given seq of entity ids, return the real entities" - [db ent-ids] - (let [indices (last (:timestamped db)) - eavt (:EAVT indices)] - (map #(% eavt) ent-ids))) - -(defn entities-by-AV - [db attr-name val-pred] - (let [indices (last (:timestamped db)) - ve (get-in indices [:AVET :test/machine] ) - relevant-entries (filter #(val-pred (first %)) ve) - relevant-ids-sets (map second relevant-entries) - relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) - ](entities-of-ids db relevant-ent-ids))) - -(defn entities-by-A - [db attr-name] - (entities-by-AV db attr-name #(= % %))) - -(defn ref-to-as - "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute - named attr-name whose type is :db/ref and the value was ent-id), all this at a given time" - ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) - ([db ent-id attr-name ts] - (let [indices ((:timestamped db) ts) - reffing-ids (get-in indices [:VAET ent-id attr-name])] - (map #(get-in indices [:EAVT %]) reffing-ids )))) - (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" [db ent-id attr-name] @@ -420,5 +394,35 @@ (defn db-before "How the db was before a given timestamp" [db ts] - (let [indices-before (subvec (:timestamped db) 0 ts )] - (assoc db :timestamped indices-before :curr-time ts))) + (let [timestamped-before (subvec (:timestamped db) 0 ts )] + (assoc db :timestamped timestamped-before :curr-time ts))) +;; (defn entities-of-ids +;; "for a given seq of entity ids, return the real entities" +;; [db ent-ids] +;; (let [indices (last (:timestamped db)) +;; eavt (:EAVT indices)] +;; (map #(% eavt) ent-ids))) + +;; (defn entities-by-AV +;; [db attr-name val-pred] +;; (let [indices (last (:timestamped db)) +;; ve (get-in indices [:AVET :test/machine] ) +;; relevant-entries (filter #(val-pred (first %)) ve) +;; relevant-ids-sets (map second relevant-entries) +;; relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) +;; ](entities-of-ids db relevant-ent-ids))) + +;; (defn entities-by-A +;; [db attr-name] +;; (entities-by-AV db attr-name #(= % %))) + +;; (defn ref-to-as +;; "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute +;; named attr-name whose type is :db/ref and the value was ent-id), all this at a given time" +;; ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) +;; ([db ent-id attr-name ts] +;; (let [indices ((:timestamped db) ts) +;; reffing-ids (get-in indices [:VAET ent-id attr-name])] +;; (map #(get-in indices [:EAVT %]) reffing-ids )))) + + diff --git a/functionalDB/src/core/storage.clj b/functionalDB/src/core/storage.clj new file mode 100644 index 000000000..e3df1a57f --- /dev/null +++ b/functionalDB/src/core/storage.clj @@ -0,0 +1,10 @@ +(ns core.storage) + +(defn initial-storage [] {}) + +(defn stored-entity [storage e-id] (e-id storage)) + +(defn update-storage [storage entity] (assoc storage (:id entity) entity)) + +(defn remove-entity-from-storage [storage entity] (dissoc storage (:id entity))) + From 764f47e337fe55dd239bde7948be41eae19e8c5f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 22 Mar 2014 16:01:54 +0200 Subject: [PATCH 059/102] replacing the EVAT index with EAVT index which indexes all --- functionalDB/src/core/fdb.clj | 49 +++++++++++------------------------ 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 93983dfe7..c57a2a92e 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -9,11 +9,10 @@ ;; -- AVET structed like this: {attrName-> {attr-val -> #{holding-elems-ids}}} ;; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) ;; can be used to know who are the entities that have a specific attribute with a specific value -;; -- EVAT structed like this: {ent-id -> {attr-val -> #{attr-names}}} -;; this provides the info - for a given ent-id (E) and specificed value (V), what are the attributes that have a specific name (A) at a given time (T). Other way to understand -;; this index is that it answers the question of how a specific entity is related to a specific value. +;; -- EAVT structed like this: {ent-id -> {attr-name -> #{attr-vals}}} mimics the storage structure +;; (defrecord Database [timestamped top-id curr-time]) -(defrecord Timestamped [storage VAET AVET EVAT]) +(defrecord Timestamped [storage VAET AVET EAVT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) @@ -84,9 +83,9 @@ (let [attr-id (keyword (:name attr))] (assoc-in ent [:attrs attr-id] attr))) -(defn ave [e a v] [a v e] ) -(defn vae [e a v] [v a e] ) -(defn eva [e a v] [e v a] ) +(defn ave [e a v] [a v e]) +(defn vae [e a v] [v a e]) +(defn eav [e a v] [e a v]) (defn ave-to-eav [a v e] [e a v]) @@ -167,8 +166,8 @@ new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) );(assoc (:storage timestamped) ent-id (update-creation-ts fixed-ent new-ts) ) new-vaet (add-entity-to-index (:VAET timestamped) ent vae ref?) new-avet (add-entity-to-index (:AVET timestamped) ent ave indexed?) - new-evat (add-entity-to-index (:EVAT timestamped) ent eva indexed?) - new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EVAT new-evat) + new-eavt (add-entity-to-index (:EAVT timestamped) ent eva indexed?) + new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EAVT new-eavt) ](assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top))) @@ -182,10 +181,10 @@ timestamped (last (:timestamped db)) vaet (remove-entity-from-index (:VAET timestamped) ent vae ref? ) avet (remove-entity-from-index (:AVET timestamped) ent ave indexed?) - evat (remove-entity-from-index (:EVAT timestamped) ent eva indexed?) + eavt (remove-entity-from-index (:EAVT timestamped) ent eav indexed?) new-storage (remove-entity-from-storage (:storage timestamped) ent) ;(dissoc (:storage timestamped) ent-id) ; removing the entity new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity - new-timestamped (assoc timestamped :storage new-storage :VAET new-vaet :AVET avet :EVAT evat) + new-timestamped (assoc timestamped :storage new-storage :VAET new-vaet :AVET avet :EAVT eavt) res (assoc db :timestamped (conj (:timestamped db) new-timestamped)) ]res)) @@ -264,8 +263,8 @@ new-storage (update-storage storage (update-entity storage ent-id updated-attr)) new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation vae ref?) new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation ave indexed?) - new-evat (update-index (:EVAT timestamped) ent-id old-attr new-val operation eva indexed?)] - (assoc timestamped :storage new-storage :VAET new-vaet :AVET new-avet :EVAT new-evat))) + new-eavt (update-index (:EAVT timestamped) ent-id old-attr new-val operation eav identity)] ; always true + (assoc timestamped :storage new-storage :VAET new-vaet :AVET new-avet :EAVT new-eavt))) (defn update-datom ([db ent-id attr-name new-val] @@ -287,7 +286,7 @@ (or (= x "_") (= (first x) \?))) (defn ind-at - "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EVAT" + "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EAVT" ([db kind] (ind-at db kind (:curr-time db))) ([db kind ts] @@ -377,7 +376,8 @@ (filter #(not-empty (% 0)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items (defmacro q - "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]})" + "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). + At the moment support only filtering queries, no joins is also assumed." [db query] `(let [query# (q-clauses ~(:where query) ) [ind# from-eav# to-eav#] (choose-index ~db query#)] @@ -396,25 +396,6 @@ [db ts] (let [timestamped-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped timestamped-before :curr-time ts))) -;; (defn entities-of-ids -;; "for a given seq of entity ids, return the real entities" -;; [db ent-ids] -;; (let [indices (last (:timestamped db)) -;; eavt (:EAVT indices)] -;; (map #(% eavt) ent-ids))) - -;; (defn entities-by-AV -;; [db attr-name val-pred] -;; (let [indices (last (:timestamped db)) -;; ve (get-in indices [:AVET :test/machine] ) -;; relevant-entries (filter #(val-pred (first %)) ve) -;; relevant-ids-sets (map second relevant-entries) -;; relevant-ent-ids (seq (reduce CS/union relevant-ids-sets )) -;; ](entities-of-ids db relevant-ent-ids))) - -;; (defn entities-by-A -;; [db attr-name] -;; (entities-by-AV db attr-name #(= % %))) ;; (defn ref-to-as ;; "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute From 013127de5476092ce37e7e0d09cc2ba5052448d7 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 23 Mar 2014 12:05:11 +0200 Subject: [PATCH 060/102] structuring an index from a spreaded around functions to a record --- functionalDB/src/core/fdb.clj | 142 +++++++++++++++++----------------- 1 file changed, 71 insertions(+), 71 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index c57a2a92e..987f03e97 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -12,12 +12,24 @@ ;; -- EAVT structed like this: {ent-id -> {attr-name -> #{attr-vals}}} mimics the storage structure ;; (defrecord Database [timestamped top-id curr-time]) +(defrecord Index [db-from-eav db-to-eav db-usage-pred]) (defrecord Timestamped [storage VAET AVET EAVT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) +(defn- single? [attr] (= :db/single (:cardinality (meta attr)))) + +(defn- indexed? [attr] (:indexed (meta attr))) + +(defn- ref? [attr] (= :db/ref (:type (meta attr)))) + (defn make-db "Create an empty database" [] - (atom (Database. [(Timestamped. (initial-storage) {} {} {})] 0 0))) + (atom (Database. [(Timestamped. + (initial-storage) ; storage + (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %)) ; VAET + (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))) ; AVET + (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))))] ; EAVT + 0 0))) (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) @@ -41,14 +53,18 @@ (defn collify [x] (if (coll? x) x [x])) -(defn- single? [attr] - (= :db/single (:cardinality (meta attr)))) - -(defn- indexed? [attr] - (:indexed (meta attr))) +(defn- next-ts [db] (inc (:curr-time db))) -(defn- ref? [attr] - (= :db/ref (:type (meta attr)))) +(defn- next-id + "returns a pair composed of the id to use for the given entity and the next free running id in the database" + [db ent] + (let [top-id (:top-id db) + ent-id (:id ent) + inceased-id (inc top-id) + [id-to-use next-top] (if (= ent-id :db/no-id-yet) + [(keyword (str inceased-id)) inceased-id] + [ent-id top-id])] + [id-to-use next-top])) (defn entity-at "the entity with the given ent-id at the given time (defualts to the latest time)" @@ -83,25 +99,6 @@ (let [attr-id (keyword (:name attr))] (assoc-in ent [:attrs attr-id] attr))) -(defn ave [e a v] [a v e]) -(defn vae [e a v] [v a e]) -(defn eav [e a v] [e a v]) - -(defn ave-to-eav [a v e] [e a v]) - -(defn- next-ts [db] (inc (:curr-time db))) - -(defn- next-id - "returns a pair composed of the id to use for the given entity and the next free running id in the database" - [db ent] - (let [top-id (:top-id db) - ent-id (:id ent) - inceased-id (inc top-id) - [id-to-use next-top] (if (= ent-id :db/no-id-yet) - [(keyword (str inceased-id)) inceased-id] - [ent-id top-id])] - [id-to-use next-top])) - (defn- update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent ts-val] @@ -119,10 +116,10 @@ (assoc-in index remove-path (disj old-entries-set val-to-remove)))) (defn- remove-entries-from-index - [index attr-name ent-id order-fn path-values operation] + [index attr-name ent-id path-values operation] (if (= operation :db/add) index - (let [paths (map #(order-fn ent-id attr-name %) path-values)] + (let [paths (map #((:db-to-eav index) ent-id attr-name %) path-values)] (reduce remove-entry-from-index index paths)))) (defn- add-entry-to-index @@ -135,40 +132,40 @@ (assoc-in index update-path (conj to-be-updated-set update-value) )))) (defn- update-attr-in-index - [index ent-id attr-name order-fn target-val operation] + [index ent-id attr-name target-val operation] (let [ colled-target-val (collify target-val) - add-entry-fn (fn [indx vl] (add-entry-to-index indx (order-fn ent-id attr-name vl) operation)) + add-entry-fn (fn [indx vl] (add-entry-to-index indx ((:db-from-eav index) ent-id attr-name vl) operation)) ] (reduce add-entry-fn index colled-target-val))) (defn- add-entity-to-index - [index ent order-fn filtering-pred] + [index ent] (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) - relevant-attrs (filter #(filtering-pred %) all-attrs ) - add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) order-fn (:value attr) :db/add)) + relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) + add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add)) ] (reduce add-in-index-fn index relevant-attrs))) (defn- remove-entity-from-index - [index ent order-fn filtering-pred] + [index ent] (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) - relevant-attrs (filter #(filtering-pred %) all-attrs ) - remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:name attr) ent-id order-fn (collify (:value attr)) :db/remove)) + relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) + remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:name attr) ent-id (collify (:value attr)) :db/remove)) ] (reduce remove-from-index-fn index relevant-attrs))) ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity [db ent] (let [[ent-id next-top] (next-id db ent) - new-ts (next-ts db) - timestamped (last (:timestamped db)) - fixed-ent (assoc ent :id ent-id) - new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) );(assoc (:storage timestamped) ent-id (update-creation-ts fixed-ent new-ts) ) - new-vaet (add-entity-to-index (:VAET timestamped) ent vae ref?) - new-avet (add-entity-to-index (:AVET timestamped) ent ave indexed?) - new-eavt (add-entity-to-index (:EAVT timestamped) ent eva indexed?) - new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EAVT new-eavt) - ](assoc db :timestamped (conj (:timestamped db) new-timestamped) + new-ts (next-ts db) + timestamped (last (:timestamped db)) + fixed-ent (assoc ent :id ent-id) + new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) );(assoc (:storage timestamped) ent-id (update-creation-ts fixed-ent new-ts) ) + new-vaet (add-entity-to-index (:VAET timestamped) ent) + new-avet (add-entity-to-index (:AVET timestamped) ent) + new-eavt (add-entity-to-index (:EAVT timestamped) ent) + new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EAVT new-eavt)] + (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top))) (defn add-entities @@ -179,9 +176,9 @@ [db ent-id] (let [ent (entity-at db ent-id) timestamped (last (:timestamped db)) - vaet (remove-entity-from-index (:VAET timestamped) ent vae ref? ) - avet (remove-entity-from-index (:AVET timestamped) ent ave indexed?) - eavt (remove-entity-from-index (:EAVT timestamped) ent eav indexed?) + vaet (remove-entity-from-index (:VAET timestamped) ent) + avet (remove-entity-from-index (:AVET timestamped) ent) + eavt (remove-entity-from-index (:EAVT timestamped) ent) new-storage (remove-entity-from-storage (:storage timestamped) ent) ;(dissoc (:storage timestamped) ent-id) ; removing the entity new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity new-timestamped (assoc timestamped :storage new-storage :VAET new-vaet :AVET avet :EAVT eavt) @@ -244,14 +241,14 @@ (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller (defn- update-index - [index ent-id old-attr target-val operation order-fn update-required-pred] - (if-not (update-required-pred old-attr) + [index ent-id old-attr target-val operation] + (if-not ((:db-usage-pred index) old-attr) index (let [old-vals (:value old-attr) attr-name (:name old-attr) remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index index attr-name ent-id order-fn remove-paths operation) - updated-index (update-attr-in-index cleaned-index ent-id attr-name order-fn target-val operation)] + cleaned-index (remove-entries-from-index index attr-name ent-id remove-paths operation) + updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] updated-index))) (defn update-entity [storage e-id new-attr] @@ -261,9 +258,9 @@ [timestamped ent-id old-attr updated-attr new-val operation] (let [ storage (:storage timestamped) new-storage (update-storage storage (update-entity storage ent-id updated-attr)) - new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation vae ref?) - new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation ave indexed?) - new-eavt (update-index (:EAVT timestamped) ent-id old-attr new-val operation eav identity)] ; always true + new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation) + new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation) + new-eavt (update-index (:EAVT timestamped) ent-id old-attr new-val operation)] (assoc timestamped :storage new-storage :VAET new-vaet :AVET new-avet :EAVT new-eavt))) (defn update-datom @@ -310,15 +307,18 @@ (variable? (str (second clause-item))) `#(~(first clause-item) % ~(last clause-item)) ; was something like (> ?a 42) (variable? (str (last clause-item))) `#(~(first clause-item) ~(second clause-item) %))) ; was something like (> 42 ?a) -(defmacro q-clause [clause] - "The aim of this macro is to return for a datalog clause (a vector with three elements describing EAV) a vector of predicates to operate on - an index containing EAV, and set for that vector its metadata to be the names of the variables that the user assiged for each item in the triplet" - (loop [[frst# & rst#] clause exprs# [] metas# [] ] - (if frst# - (recur rst# (conj exprs# `(clause-item-expr ~ frst#)) (conj metas#`(clause-item-meta ~ frst#))) +(defmacro q-clause + "The aim of this macro to build for a datalog clause (a vector with three elements describing EAV) a vector of predicates that would operate on + an index, and set for that vector's metadata to be the names of the variables that the user assiged for each item in the clause" + [clause] + (loop [[frst-itm# & rst-itm#] clause exprs# [] metas# [] ] + (if frst-itm# + (recur rst-itm# (conj exprs# `(clause-item-expr ~ frst-itm#)) (conj metas#`(clause-item-meta ~ frst-itm#))) (with-meta exprs# {:db/variable metas#})))) - (defmacro q-clauses [clauses] + (defmacro q-clauses + "create a vector of queries to operate on indexes based on the given vector of clauses " + [clauses] (loop [[frst# & rst#] clauses preds-vecs# [] ] (if-not frst# preds-vecs# (recur rst# `(conj ~preds-vecs# (q-clause ~frst#))) ))) @@ -327,22 +327,22 @@ "Upon receiving a database and query clauses, this function responsible to deduce on which index in the db it is best to perfom the query clauses, and then return a vector in which the first element is the decided index and the second element is a function that knows how to restore an EAV structure from that decided index path structure." [db query] - [(ind-at db :AVET) ave ave-to-eav]) + (ind-at db :AVET)) (defn filter-index "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: -- At index 0 : the elements (e-ids or attribute names) that are found at the leave of the path. -- At index 1 : the key used at the first level of the path. -- At index 2 : the key used at the second level of the path" ; the triplet structure results in a EAV structure for the common usage - [index path-preds from-eav to-eav] + [index path-preds] (for [ path-pred path-preds - :let [[lvl1-prd lvl2-prd lvl3-prd] (apply from-eav path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta + :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (:db-from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta [k1 l2map] index ; keys and values of the first level :when (lvl1-prd k1) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate [k2 l3-set] l2map ; keys and values of the second level :when (lvl2-prd k2) ; filtering to keep only the keys and vals of keys that passed the second level predicate :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them - (with-meta (to-eav k1 k2 res) (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables + (with-meta ((:db-to-eav index) k1 k2 res) (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables (defn items-that-answer-all-conditions "takes the sequence of all the items collection, each such collection answered one condition, we test here what are the items that answered all of the conditions @@ -369,8 +369,8 @@ -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" - [index path-preds from-eav to-eav] - (let [filtered-paths (filter-index index path-preds from-eav to-eav) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path + [index path-preds] + (let [filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path relevant-elements (items-that-answer-all-conditions (map first filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths relevant-paths (map mask-path-leaf-with-items filtered-paths (repeat relevant-elements))] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates (filter #(not-empty (% 0)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items @@ -380,8 +380,8 @@ At the moment support only filtering queries, no joins is also assumed." [db query] `(let [query# (q-clauses ~(:where query) ) - [ind# from-eav# to-eav#] (choose-index ~db query#)] - (query-index ind# query# from-eav# to-eav#))) + ind# (choose-index ~db query#)] + (query-index ind# query#))) (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" From fc0a82d847c8912df51560af68c1bf2428796cf3 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 23 Mar 2014 13:42:13 +0200 Subject: [PATCH 061/102] using assoc-in when setting the creation timestamp of an entity's attributes instead of decomposing and recomposing the entity --- functionalDB/src/core/fdb.clj | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 987f03e97..cf6b99acd 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -102,11 +102,7 @@ (defn- update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent ts-val] - (let [ks (keys (:attrs ent)) - vls (vals (:attrs ent)) - updated-attrs-vals (map #(assoc % :ts ts-val) vls) - updated-attrs (zipmap ks updated-attrs-vals) - ](assoc ent :attrs updated-attrs))) + (reduce ent #(assoc-in %1 [:attrs %2 :ts ] ts-val) (keys (:attrs ent)))) (defn- remove-entry-from-index [index path] From 7829340506e636d6efa39723a57191dbe1ae627d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 24 Mar 2014 12:47:57 +0200 Subject: [PATCH 062/102] fixing wrong ordering of arguments --- functionalDB/src/core/fdb.clj | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index cf6b99acd..8e6093f79 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -31,7 +31,8 @@ (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))))] ; EAVT 0 0))) -(defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" +(defn make-entity + "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) ([id] (Entity. id {}))) @@ -102,7 +103,7 @@ (defn- update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent ts-val] - (reduce ent #(assoc-in %1 [:attrs %2 :ts ] ts-val) (keys (:attrs ent)))) + (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) (defn- remove-entry-from-index [index path] @@ -138,8 +139,8 @@ (let [ent-id (:id ent) all-attrs (vals (:attrs ent)) relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) - add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add)) - ] (reduce add-in-index-fn index relevant-attrs))) + add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] + (reduce add-in-index-fn index relevant-attrs))) (defn- remove-entity-from-index [index ent] @@ -156,10 +157,10 @@ new-ts (next-ts db) timestamped (last (:timestamped db)) fixed-ent (assoc ent :id ent-id) - new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) );(assoc (:storage timestamped) ent-id (update-creation-ts fixed-ent new-ts) ) - new-vaet (add-entity-to-index (:VAET timestamped) ent) - new-avet (add-entity-to-index (:AVET timestamped) ent) - new-eavt (add-entity-to-index (:EAVT timestamped) ent) + new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) ) + new-vaet (add-entity-to-index (:VAET timestamped) fixed-ent) + new-avet (add-entity-to-index (:AVET timestamped) fixed-ent) + new-eavt (add-entity-to-index (:EAVT timestamped) fixed-ent) new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EAVT new-eavt)] (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top))) @@ -187,11 +188,13 @@ (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [initial-timestamped (:timestamped initial-db) + stam (println 10) new-timestamped (last (:timestamped transacted)) - res (assoc initial-db - :timestamped (conj initial-timestamped new-timestamped) - :curr-time (next-ts initial-db) - :top-id (:top-id transacted))] + stam (println 11) + res (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) + :curr-time (next-ts initial-db) :top-id (:top-id transacted)) + stam (println 12) + ] res)))) (defmacro _transact @@ -286,12 +289,12 @@ (kind ((:timestamped db) ts)))) (defmacro clause-item-meta - "Finds the name of the variable at an item of a datalog clause element. If no variable, returning :db/no-var" + "Finds the name of the variable at an item of a datalog clause element. If no variable, returning nil" [clause-item] (cond - (coll? clause-item) (first (filter #(variable? %) (map #(str %) clause-item))) ; the item is an s-expression, need to treat it as a coll, by going over it and returning the name of the variable + (coll? clause-item) (first (filter variable? (map str clause-item))) ; the item is an s-expression, need to treat it as a coll, by going over it and returning the name of the variable (variable? (str clause-item)) (str clause-item) ; the item is a simple variable - :no-variable-in-clause :db/no-var)) ; the item is a value and not a variable + :no-variable-in-clause nil)) ; the item is a value and not a variable (defmacro clause-item-expr "Create a predicate for each element in the datalog clause" From 32452920ebc6152572b53803d3c3a1fdfe16b154 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 24 Mar 2014 13:15:10 +0200 Subject: [PATCH 063/102] removing prints --- functionalDB/src/core/fdb.clj | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 8e6093f79..5a7ef86c7 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -188,13 +188,9 @@ (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [initial-timestamped (:timestamped initial-db) - stam (println 10) new-timestamped (last (:timestamped transacted)) - stam (println 11) res (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) - :curr-time (next-ts initial-db) :top-id (:top-id transacted)) - stam (println 12) - ] + :curr-time (next-ts initial-db) :top-id (:top-id transacted))] res)))) (defmacro _transact From 0992d0dea4a51c4787cb35e97f78b979251ea90b Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 24 Mar 2014 20:00:37 +0200 Subject: [PATCH 064/102] refactoring the add flow towards not knowing hardcodedly about the indices --- functionalDB/src/core/fdb.clj | 50 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 26 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5a7ef86c7..cc388415c 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -12,7 +12,7 @@ ;; -- EAVT structed like this: {ent-id -> {attr-name -> #{attr-vals}}} mimics the storage structure ;; (defrecord Database [timestamped top-id curr-time]) -(defrecord Index [db-from-eav db-to-eav db-usage-pred]) +(defrecord Index [db-from-eav db-to-eav db-usage-pred db-laef-index]) (defrecord Timestamped [storage VAET AVET EAVT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) @@ -26,9 +26,9 @@ (defn make-db "Create an empty database" [] (atom (Database. [(Timestamped. (initial-storage) ; storage - (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %)) ; VAET - (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))) ; AVET - (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))))] ; EAVT + (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins + (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))0) ; AVET - for filtering + (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))) 2)] ; EAVT - for filtering 0 0))) (defn make-entity @@ -134,13 +134,30 @@ add-entry-fn (fn [indx vl] (add-entry-to-index indx ((:db-from-eav index) ent-id attr-name vl) operation)) ] (reduce add-entry-fn index colled-target-val))) -(defn- add-entity-to-index - [index ent] +(defn add-entity-to-index + [ent timestamped ind-name] (let [ent-id (:id ent) + index (ind-name timestamped) all-attrs (vals (:attrs ent)) relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] - (reduce add-in-index-fn index relevant-attrs))) + (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) + +(denf fix-new-entity [db ent] + (let [[ent-id next-top-id] (next-id db ent) + new-ts (next-ts db)] + (update-creation-ts (assoc ent :id ent-id) new-ts))) + +;when adding an entity, its attributes' timestamp would be set to be the current one +(defn add-entity + [db ent] + (let [fixed-ent (fix-new-entity db ent) + new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) + add-fn (partial add-entity-to-index fixed-ent) + new-timestamped (reduce add-fn new-timestamp [:VAET :AVET :EAVT])] + (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top))) + +(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) (defn- remove-entity-from-index [index ent] @@ -150,25 +167,6 @@ remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:name attr) ent-id (collify (:value attr)) :db/remove)) ] (reduce remove-from-index-fn index relevant-attrs))) -;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity - [db ent] - (let [[ent-id next-top] (next-id db ent) - new-ts (next-ts db) - timestamped (last (:timestamped db)) - fixed-ent (assoc ent :id ent-id) - new-storage (update-storage (:storage timestamped) (update-creation-ts fixed-ent new-ts) ) - new-vaet (add-entity-to-index (:VAET timestamped) fixed-ent) - new-avet (add-entity-to-index (:AVET timestamped) fixed-ent) - new-eavt (add-entity-to-index (:EAVT timestamped) fixed-ent) - new-timestamped (assoc timestamped :VAET new-vaet :storage new-storage :AVET new-avet :EAVT new-eavt)] - (assoc db :timestamped (conj (:timestamped db) new-timestamped) - :top-id next-top))) - -(defn add-entities - [db ents-seq] - (reduce add-entity db ents-seq)) - (defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) From ce190287541c4e80270ab2a89fdcd5636d07a30f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 26 Mar 2014 15:44:44 +0200 Subject: [PATCH 065/102] cleaner removal process --- functionalDB/src/core/fdb.clj | 87 ++++++++++++++++++----------------- 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index cc388415c..07e8e6cb4 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -12,7 +12,7 @@ ;; -- EAVT structed like this: {ent-id -> {attr-name -> #{attr-vals}}} mimics the storage structure ;; (defrecord Database [timestamped top-id curr-time]) -(defrecord Index [db-from-eav db-to-eav db-usage-pred db-laef-index]) +(defrecord Index [db-from-eav db-to-eav db-usage-pred db-leaf-index]) (defrecord Timestamped [storage VAET AVET EAVT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) @@ -28,7 +28,7 @@ (initial-storage) ; storage (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))0) ; AVET - for filtering - (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))) 2)] ; EAVT - for filtering + (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))2) )] ; EAVT - for filtering 0 0))) (defn make-entity @@ -105,20 +105,6 @@ [ent ts-val] (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) -(defn- remove-entry-from-index - [index path] - (let [remove-path (butlast path) - val-to-remove (last path) - old-entries-set (get-in index remove-path )] - (assoc-in index remove-path (disj old-entries-set val-to-remove)))) - -(defn- remove-entries-from-index - [index attr-name ent-id path-values operation] - (if (= operation :db/add) - index - (let [paths (map #((:db-to-eav index) ent-id attr-name %) path-values)] - (reduce remove-entry-from-index index paths)))) - (defn- add-entry-to-index [index path operation] (if (= operation :db/remove) @@ -143,42 +129,59 @@ add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) -(denf fix-new-entity [db ent] +(defn fix-new-entity [db ent] (let [[ent-id next-top-id] (next-id db ent) - new-ts (next-ts db)] - (update-creation-ts (assoc ent :id ent-id) new-ts))) + new-ts (next-ts db)] + [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) ;when adding an entity, its attributes' timestamp would be set to be the current one (defn add-entity [db ent] - (let [fixed-ent (fix-new-entity db ent) + (let [[fixed-ent next-top-id](fix-new-entity db ent) new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) add-fn (partial add-entity-to-index fixed-ent) - new-timestamped (reduce add-fn new-timestamp [:VAET :AVET :EAVT])] - (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top))) + new-timestamped (reduce add-fn new-timestamped [:VAET :AVET :EAVT])] + (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) (defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) +(defn remove-entry-from-index + [index path] + (let [path-head (first path) + path-to-items (butlast path) + val-to-remove (last path) + old-entries-set (get-in index path-to-items)] + (cond + (and (= 1 (count old-entries-set) ) (= 1 (count (path-head index)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely + (or (nil? old-entries-set)(= (count old-entries-set) 1)) (update-in index [path-head] dissoc (second path)) ; a path that spreads at the second item - just remove the unneeded part of it + :else (update-in index path-to-items disj val-to-remove)))) + +(defn- remove-entries-from-index + [ent-id operation index attr] + (if (= operation :db/add) + index + (let [attr-name (:name attr) + datom-vals (collify (:value attr)) + paths (map #((:db-to-eav index) ent-id attr-name %) datom-vals)] + (reduce remove-entry-from-index index paths)))) + (defn- remove-entity-from-index - [index ent] + [ent timestamped ind-name] (let [ent-id (:id ent) + index (ind-name timestamped) all-attrs (vals (:attrs ent)) relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) - remove-from-index-fn (fn [ind attr] ( remove-entries-from-index ind (:name attr) ent-id (collify (:value attr)) :db/remove)) - ] (reduce remove-from-index-fn index relevant-attrs))) + remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] + (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) (defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) - timestamped (last (:timestamped db)) - vaet (remove-entity-from-index (:VAET timestamped) ent) - avet (remove-entity-from-index (:AVET timestamped) ent) - eavt (remove-entity-from-index (:EAVT timestamped) ent) - new-storage (remove-entity-from-storage (:storage timestamped) ent) ;(dissoc (:storage timestamped) ent-id) ; removing the entity - new-vaet (dissoc vaet ent-id) ; removing incoming REFs to the entity - new-timestamped (assoc timestamped :storage new-storage :VAET new-vaet :AVET avet :EAVT eavt) - res (assoc db :timestamped (conj (:timestamped db) new-timestamped)) - ]res)) + timestamped (update-in (last (:timestamped db)) [:VAET] dissoc ent-id) + new-timestamped (assoc timestamped :storage (remove-entity-from-storage (:storage timestamped) ent)) + remove-fn (partial remove-entity-from-index ent) + new-timestamped (reduce remove-fn new-timestamped [:VAET :AVET :EAVT])] + (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) (defn transact-on-db [initial-db txs] @@ -240,20 +243,20 @@ (let [old-vals (:value old-attr) attr-name (:name old-attr) remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index index attr-name ent-id remove-paths operation) + cleaned-index (remove-entries-from-index ent-id operation index old-attr) ;ent-id operation index attr updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] updated-index))) (defn update-entity [storage e-id new-attr] - (assoc-in (stored-entity storage ent-id) [:attrs (:name new-attr)])) + (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) (defn- update-timestamped [timestamped ent-id old-attr updated-attr new-val operation] - (let [ storage (:storage timestamped) - new-storage (update-storage storage (update-entity storage ent-id updated-attr)) - new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation) - new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation) - new-eavt (update-index (:EAVT timestamped) ent-id old-attr new-val operation)] + (let [storage (:storage timestamped) + new-storage (update-storage storage (update-entity storage ent-id updated-attr)) + new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation) + new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation) + new-eavt (update-index (:EAVT timestamped) ent-id old-attr new-val operation)] (assoc timestamped :storage new-storage :VAET new-vaet :AVET new-avet :EAVT new-eavt))) (defn update-datom @@ -265,7 +268,7 @@ attr (attr-at db ent-id attr-name) updated-attr (update-attr attr new-val update-ts operation) fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation) - new-db (update-in db [:timestamped] conj fully-updated-timestamped)] + new-db (update-in db [:timestamped] conj fully-updated-timestamped)] new-db))) ;;;;;;;;;;;;;;;;;;;;;;; queries From d0e900d67cb5dbab730400e6dae77b8263c4f639 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 26 Mar 2014 15:46:21 +0200 Subject: [PATCH 066/102] cleanup --- functionalDB/src/core/manage.clj | 5 ++--- functionalDB/src/core/storage.clj | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index 3c42af37d..c3c387b2b 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -5,10 +5,9 @@ (def __ALL-DBS__ (atom {})) (defn _get-db [dbs db-name] - (if (db-name dbs ) dbs (assoc dbs db-name (make-db)))) + (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) -(defn _reset-db[dbs db-name] - (dissoc dbs db-name)) +(defn _reset-db[dbs db-name] (dissoc dbs db-name)) (defn _as-db-name [db-name] (keyword db-name)) diff --git a/functionalDB/src/core/storage.clj b/functionalDB/src/core/storage.clj index e3df1a57f..99f699d62 100644 --- a/functionalDB/src/core/storage.clj +++ b/functionalDB/src/core/storage.clj @@ -7,4 +7,3 @@ (defn update-storage [storage entity] (assoc storage (:id entity) entity)) (defn remove-entity-from-storage [storage entity] (dissoc storage (:id entity))) - From e449eefb9e679099b05051603d5c39e3f4732236 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 26 Mar 2014 22:24:04 +0200 Subject: [PATCH 067/102] refactoring out the knowledge about the names of indices from the update process --- functionalDB/src/core/fdb.clj | 34 ++++++++++++++++---------------- functionalDB/src/core/manage.clj | 6 +++--- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 07e8e6cb4..537fc96f9 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -31,6 +31,8 @@ (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))2) )] ; EAVT - for filtering 0 0))) +(defn indices[] [:VAET :AVET :EAVT]) + (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" ([] (make-entity :db/no-id-yet )) @@ -140,7 +142,7 @@ (let [[fixed-ent next-top-id](fix-new-entity db ent) new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) add-fn (partial add-entity-to-index fixed-ent) - new-timestamped (reduce add-fn new-timestamped [:VAET :AVET :EAVT])] + new-timestamped (reduce add-fn new-timestamped (indices))] (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) (defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) @@ -152,8 +154,9 @@ val-to-remove (last path) old-entries-set (get-in index path-to-items)] (cond + (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here (and (= 1 (count old-entries-set) ) (= 1 (count (path-head index)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely - (or (nil? old-entries-set)(= (count old-entries-set) 1)) (update-in index [path-head] dissoc (second path)) ; a path that spreads at the second item - just remove the unneeded part of it + (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it :else (update-in index path-to-items disj val-to-remove)))) (defn- remove-entries-from-index @@ -162,7 +165,7 @@ index (let [attr-name (:name attr) datom-vals (collify (:value attr)) - paths (map #((:db-to-eav index) ent-id attr-name %) datom-vals)] + paths (map #((:db-from-eav index) ent-id attr-name %) datom-vals)] (reduce remove-entry-from-index index paths)))) (defn- remove-entity-from-index @@ -180,7 +183,7 @@ timestamped (update-in (last (:timestamped db)) [:VAET] dissoc ent-id) new-timestamped (assoc timestamped :storage (remove-entity-from-storage (:storage timestamped) ent)) remove-fn (partial remove-entity-from-index ent) - new-timestamped (reduce remove-fn new-timestamped [:VAET :AVET :EAVT])] + new-timestamped (reduce remove-fn new-timestamped (indices))] (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) (defn transact-on-db @@ -237,15 +240,16 @@ (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller (defn- update-index - [index ent-id old-attr target-val operation] - (if-not ((:db-usage-pred index) old-attr) - index - (let [old-vals (:value old-attr) + [ent-id old-attr target-val operation timestamped ind-name] + (if-not ((get-in timestamped [ind-name :db-usage-pred]) old-attr) + timestamped + (let [index (ind-name timestamped) + old-vals (:value old-attr) attr-name (:name old-attr) remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index ent-id operation index old-attr) ;ent-id operation index attr + cleaned-index (remove-entries-from-index ent-id operation index old-attr) updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] - updated-index))) + (assoc timestamped ind-name updated-index)))) (defn update-entity [storage e-id new-attr] (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) @@ -253,11 +257,8 @@ (defn- update-timestamped [timestamped ent-id old-attr updated-attr new-val operation] (let [storage (:storage timestamped) - new-storage (update-storage storage (update-entity storage ent-id updated-attr)) - new-vaet (update-index (:VAET timestamped) ent-id old-attr new-val operation) - new-avet (update-index (:AVET timestamped) ent-id old-attr new-val operation) - new-eavt (update-index (:EAVT timestamped) ent-id old-attr new-val operation)] - (assoc timestamped :storage new-storage :VAET new-vaet :AVET new-avet :EAVT new-eavt))) + new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] + (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) (defn update-datom ([db ent-id attr-name new-val] @@ -360,8 +361,7 @@ (defn query-index "an index is a 3 level map (VAE, AVE, EVA) that is found at a specific time (T). The path-preds argument is a seq of vectors. Each of these vectors contains two elements: - -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, - E -> an id matching predicator) + -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, E -> an id matching predicator) -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index c3c387b2b..a58ebfc78 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,6 +1,6 @@ -;; management of db and db connections -;; via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection -(ns core.manage (:use core.fdb)) +(ns core.manage + "Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" + (:use core.fdb)) (def __ALL-DBS__ (atom {})) From 79392d089a0bc77a33fa250c989717457fcbc6a9 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 28 Mar 2014 10:27:52 +0300 Subject: [PATCH 068/102] cleaning back reffs when removing an entity --- functionalDB/src/core/fdb.clj | 126 ++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 58 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 537fc96f9..5e4df3b9c 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -136,8 +136,7 @@ new-ts (next-ts db)] [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) -;when adding an entity, its attributes' timestamp would be set to be the current one -(defn add-entity +(defn add-entity ;when adding an entity, its attributes' timestamp would be set to be the current one [db ent] (let [[fixed-ent next-top-id](fix-new-entity db ent) new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) @@ -147,6 +146,59 @@ (defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) +(defn- update-attr-modification-time + [attr new-ts] + (assoc attr :ts new-ts :prev-ts ( :ts attr))) + +(defn- update-attr + [attr new-val new-ts operation] + {:pre [(if (single? attr) + (= operation :db/reset-to) + (contains? #{:db/reset-to :db/add :db/remove} operation))]} + (-> attr + (update-attr-modification-time new-ts) + (update-attr-value new-val operation))) + +(defn- remove-path-values + [old-vals target-val operation] + (cond + (= operation :db/add) [] ; nothing to remove + (= operation :db/reset-to) old-vals ; removing all of the old values + (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller + +(defn- update-index + [ent-id old-attr target-val operation timestamped ind-name] + (if-not ((get-in timestamped [ind-name :db-usage-pred]) old-attr) + timestamped + (let [index (ind-name timestamped) + old-vals (:value old-attr) + attr-name (:name old-attr) + remove-paths (remove-path-values old-vals target-val operation) + cleaned-index (remove-entries-from-index ent-id operation index old-attr) + updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] + (assoc timestamped ind-name updated-index)))) + +(defn update-entity [storage e-id new-attr] + (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) + +(defn- update-timestamped + [timestamped ent-id old-attr updated-attr new-val operation] + (let [storage (:storage timestamped) + new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] + (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) + +(defn update-datom + ([db ent-id attr-name new-val] + (update-datom db ent-id attr-name new-val :db/reset-to )) + ([db ent-id attr-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) + (let [update-ts (next-ts db) + timestamped (last (:timestamped db)) + attr (attr-at db ent-id attr-name) + updated-attr (update-attr attr new-val update-ts operation) + fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation) + new-db (update-in db [:timestamped] conj fully-updated-timestamped)] + new-db))) + (defn remove-entry-from-index [index path] (let [path-head (first path) @@ -177,10 +229,23 @@ remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) +(defn reffing-datoms-to[e-id timestamped] + (let [vaet (:VAET timestamped)] + (for [[attr-name reffing-set] (e-id vaet) + reffing reffing-set] + [reffing attr-name e-id]))) + +(defn remove-back-refs [db e-id timestamped] + (let [refing-datoms (reffing-datoms-to e-id timestamped) + remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) + clean-db (reduce remove-fn db refing-datoms)] + (last (:timestamped db)))) + (defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) - timestamped (update-in (last (:timestamped db)) [:VAET] dissoc ent-id) + timestamped (remove-back-refs db ent-id (last (:timestamped db))) + timestamped (update-in timestamped [:VAET] dissoc ent-id) new-timestamped (assoc timestamped :storage (remove-entity-from-storage (:storage timestamped) ent)) remove-fn (partial remove-entity-from-index ent) new-timestamped (reduce remove-fn new-timestamped (indices))] @@ -217,61 +282,6 @@ [db & txs] `(_transact ~db swap! ~@txs)) -;;;;;;;;;;;;;;;;;;;;;;; datom updates - -(defn- update-attr-modification-time - [attr new-ts] - (assoc attr :ts new-ts :prev-ts ( :ts attr))) - -(defn- update-attr - [attr new-val new-ts operation] - {:pre [(if (single? attr) - (= operation :db/reset-to) - (contains? #{:db/reset-to :db/add :db/remove} operation))]} - (-> attr - (update-attr-modification-time new-ts) - (update-attr-value new-val operation))) - -(defn- remove-path-values - [old-vals target-val operation] - (cond - (= operation :db/add) [] ; nothing to remove - (= operation :db/reset-to) old-vals ; removing all of the old values - (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller - -(defn- update-index - [ent-id old-attr target-val operation timestamped ind-name] - (if-not ((get-in timestamped [ind-name :db-usage-pred]) old-attr) - timestamped - (let [index (ind-name timestamped) - old-vals (:value old-attr) - attr-name (:name old-attr) - remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index ent-id operation index old-attr) - updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] - (assoc timestamped ind-name updated-index)))) - -(defn update-entity [storage e-id new-attr] - (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) - -(defn- update-timestamped - [timestamped ent-id old-attr updated-attr new-val operation] - (let [storage (:storage timestamped) - new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] - (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) - -(defn update-datom - ([db ent-id attr-name new-val] - (update-datom db ent-id attr-name new-val :db/reset-to )) - ([db ent-id attr-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) - (let [update-ts (next-ts db) - timestamped (last (:timestamped db)) - attr (attr-at db ent-id attr-name) - updated-attr (update-attr attr new-val update-ts operation) - fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation) - new-db (update-in db [:timestamped] conj fully-updated-timestamped)] - new-db))) - ;;;;;;;;;;;;;;;;;;;;;;; queries (defn variable? From f85e373ecc6651621070de558a4ed75807911a61 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 28 Mar 2014 16:08:31 +0300 Subject: [PATCH 069/102] handling the removal of back refs in case of single attributes --- functionalDB/src/core/fdb.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5e4df3b9c..560851213 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -153,7 +153,7 @@ (defn- update-attr [attr new-val new-ts operation] {:pre [(if (single? attr) - (= operation :db/reset-to) + (contains? #{:db/reset-to :db/remove} operation) (contains? #{:db/reset-to :db/add :db/remove} operation))]} (-> attr (update-attr-modification-time new-ts) From 6ff36bed98342b7b1e3f0bcde3e46f4549e51274 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 30 Mar 2014 16:45:40 +0300 Subject: [PATCH 070/102] cleaning out the AVET residuals from the query results structuring process --- functionalDB/src/core/fdb.clj | 39 ++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 560851213..6af8e2a88 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -26,9 +26,9 @@ (defn make-db "Create an empty database" [] (atom (Database. [(Timestamped. (initial-storage) ; storage - (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins - (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))0) ; AVET - for filtering - (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))2) )] ; EAVT - for filtering + (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 2) ; VAET - for graph qeries and joins + (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))2) ; AVET - for filtering + (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))0) )] ; EAVT - for filtering 0 0))) (defn indices[] [:VAET :AVET :EAVT]) @@ -190,7 +190,7 @@ (defn update-datom ([db ent-id attr-name new-val] (update-datom db ent-id attr-name new-val :db/reset-to )) - ([db ent-id attr-name new-val operation ] ; operation may be either :db/reset-to :db/add ,or :db/remove (the last two are valid only if the attr cardinality is :db/multiple) + ([db ent-id attr-name new-val operation ] (let [update-ts (next-ts db) timestamped (last (:timestamped db)) attr (attr-at db ent-id attr-name) @@ -211,7 +211,7 @@ (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it :else (update-in index path-to-items disj val-to-remove)))) -(defn- remove-entries-from-index +(defn remove-entries-from-index [ent-id operation index attr] (if (= operation :db/add) index @@ -271,16 +271,13 @@ (list* (conj res# accum-txs#)))))) (defn _what-if + "Operates on the db with the given transactions, but without eventually updating it" [ db f txs] (f db txs)) -(defmacro what-if - [db & txs] - `(_transact ~db _what-if ~@txs)) +(defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) -(defmacro transact - [db & txs] - `(_transact ~db swap! ~@txs)) +(defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) ;;;;;;;;;;;;;;;;;;;;;;; queries @@ -349,7 +346,7 @@ [k2 l3-set] l2map ; keys and values of the second level :when (lvl2-prd k2) ; filtering to keep only the keys and vals of keys that passed the second level predicate :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them - (with-meta ((:db-to-eav index) k1 k2 res) (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables + (with-meta [k1 k2 res] (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables (defn items-that-answer-all-conditions "takes the sequence of all the items collection, each such collection answered one condition, we test here what are the items that answered all of the conditions @@ -365,8 +362,9 @@ (defn mask-path-leaf-with-items "a path is a vector triplet, at the first position there's a set of the path's items, here we intersect the path's items with the set of relevant items" - [path relevant-items] - (assoc path 0 (CS/intersection relevant-items (path 0)))) + [index relevant-items path] + (let [leaf-ind (:db-leaf-index index )] + (assoc path leaf-ind (CS/intersection relevant-items (path leaf-ind ))))) (defn query-index "an index is a 3 level map (VAE, AVE, EVA) that is found at a specific time (T). @@ -376,17 +374,20 @@ -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" [index path-preds] - (let [filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path - relevant-elements (items-that-answer-all-conditions (map first filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths - relevant-paths (map mask-path-leaf-with-items filtered-paths (repeat relevant-elements))] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates - (filter #(not-empty (% 0)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items + (let [leaf-ind (:db-leaf-index index) + filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path + relevant-items (items-that-answer-all-conditions (map #(get % leaf-ind ) filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths + relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates + (filter #(not-empty (% leaf-ind)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items (defmacro q "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). At the moment support only filtering queries, no joins is also assumed." [db query] `(let [query# (q-clauses ~(:where query) ) - ind# (choose-index ~db query#)] + ind# (choose-index ~db query#) + + ] (query-index ind# query#))) (defn evolution-of From 198b489a1ba6b50ffcf93a875c661611a563ef9a Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 1 Apr 2014 13:46:31 +0300 Subject: [PATCH 071/102] preparing query results toward reporting --- functionalDB/src/core/fdb.clj | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 6af8e2a88..29741a9bb 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -363,8 +363,22 @@ (defn mask-path-leaf-with-items "a path is a vector triplet, at the first position there's a set of the path's items, here we intersect the path's items with the set of relevant items" [index relevant-items path] - (let [leaf-ind (:db-leaf-index index )] - (assoc path leaf-ind (CS/intersection relevant-items (path leaf-ind ))))) + (update-in path [2] CS/intersection relevant-items ));) + + (defn seqify-result-path + "A result-path is a path whose leaves are the items representing the items of the query chaining variable. This function + returns for a result path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item is followed + by it's variable name as was inserted in the query" + [index path ] + (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] + meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) + all-path (interleave seq-path meta-path)] + (apply (partial map vector) all-path))) + +(defn merge-query-and-meta [q-res index] + (let [seq-res-path (mapcat (partial seqify-result-path index) q-res ) + reversed-res-path (map reverse seq-res-path)] + (reduce #(assoc-in %1 (butlast %2) (last %2)) {} reversed-res-path))) (defn query-index "an index is a 3 level map (VAE, AVE, EVA) that is found at a specific time (T). @@ -374,11 +388,11 @@ -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" [index path-preds] - (let [leaf-ind (:db-leaf-index index) + (let [;leaf-ind (:db-leaf-index index) filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path - relevant-items (items-that-answer-all-conditions (map #(get % leaf-ind ) filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths + relevant-items (items-that-answer-all-conditions (map last filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates - (filter #(not-empty (% leaf-ind)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items + (filter #(not-empty (last %)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items (defmacro q "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). @@ -386,9 +400,10 @@ [db query] `(let [query# (q-clauses ~(:where query) ) ind# (choose-index ~db query#) - - ] - (query-index ind# query#))) + q-res# (query-index ind# query#) + ;res# (merge-query-and-meta q-res# ind#) + ]q-res# + )) (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" From bb6eaf5338bd4589101bdfdd039d4374b51d9b0d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 1 Apr 2014 13:54:10 +0300 Subject: [PATCH 072/102] returning the right query result tree towards reporting as the user defined --- functionalDB/src/core/fdb.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 29741a9bb..626a74f4a 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -401,8 +401,8 @@ `(let [query# (q-clauses ~(:where query) ) ind# (choose-index ~db query#) q-res# (query-index ind# query#) - ;res# (merge-query-and-meta q-res# ind#) - ]q-res# + res# (merge-query-and-meta q-res# ind#) + ]res# )) (defn evolution-of From 90d0fecfe1001272eafc4839c5f2398c2bd11853 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Thu, 3 Apr 2014 15:24:46 +0300 Subject: [PATCH 073/102] presenting the query results --- functionalDB/src/core/fdb.clj | 40 +++++++++++------ functionalDB/src/core/manage.clj | 2 +- functionalDB/src/scenarios/hospital.clj | 59 +++++++++++++++++++------ 3 files changed, 74 insertions(+), 27 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 626a74f4a..f30680789 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -26,9 +26,9 @@ (defn make-db "Create an empty database" [] (atom (Database. [(Timestamped. (initial-storage) ; storage - (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 2) ; VAET - for graph qeries and joins - (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))2) ; AVET - for filtering - (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))0) )] ; EAVT - for filtering + (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins + (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))0) ; AVET - for filtering + (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))2) )] ; EAVT - for filtering 0 0))) (defn indices[] [:VAET :AVET :EAVT]) @@ -335,9 +335,10 @@ (defn filter-index "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: - -- At index 0 : the elements (e-ids or attribute names) that are found at the leave of the path. - -- At index 1 : the key used at the first level of the path. - -- At index 2 : the key used at the second level of the path" ; the triplet structure results in a EAV structure for the common usage + -- At index 0 : the key used at the first level of the path. + -- At index 1 : the key used at the second level of the path + -- At index 2 : the elements (e-ids or attribute names) that are found at the leave of the path." + [index path-preds] (for [ path-pred path-preds :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (:db-from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta @@ -371,14 +372,15 @@ by it's variable name as was inserted in the query" [index path ] (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] - meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) - all-path (interleave seq-path meta-path)] + meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index + all-path (interleave seq-path meta-path)] (apply (partial map vector) all-path))) (defn merge-query-and-meta [q-res index] (let [seq-res-path (mapcat (partial seqify-result-path index) q-res ) - reversed-res-path (map reverse seq-res-path)] - (reduce #(assoc-in %1 (butlast %2) (last %2)) {} reversed-res-path))) + reversed-res-path (map reverse seq-res-path) + paired-path-meta (map (partial partition 2) reversed-res-path)] + (reduce #(assoc-in %1 (butlast %2) (last %2)) {} paired-path-meta))) (defn query-index "an index is a 3 level map (VAE, AVE, EVA) that is found at a specific time (T). @@ -388,12 +390,22 @@ -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" [index path-preds] - (let [;leaf-ind (:db-leaf-index index) - filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path + (let [filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path relevant-items (items-that-answer-all-conditions (map last filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates (filter #(not-empty (last %)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items +(defn values-of-vars + "For a given query result (a merge of the result and the metadata), returns the variable values found at the requested variable set" + [vars-set q-res] + (let [[root-pair child-map] q-res ; breaking the result to have both the root pair and all the rest of the pairs + united-children (CS/union (set (keys child-map)) (set (vals child-map))) ; uniting the child pairs to one set of bindings + all-binding-set (conj united-children root-pair) ; adding to the set of bindings the root pair + relevant-binds (filter #(contains? vars-set (first %)) all-binding-set)] ; keeping only the relevant pairs + (zipmap (map first relevant-binds) (map second relevant-binds))) ) ; turning the pairs into map + +(defmacro settify [coll] (set (map str coll))) + (defmacro q "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). At the moment support only filtering queries, no joins is also assumed." @@ -401,7 +413,9 @@ `(let [query# (q-clauses ~(:where query) ) ind# (choose-index ~db query#) q-res# (query-index ind# query#) - res# (merge-query-and-meta q-res# ind#) + merged-res# (merge-query-and-meta q-res# ind#) + find-items# (settify ~(:find query)) + res# (map (partial values-of-vars (set find-items#) ) merged-res# ) ]res# )) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index a58ebfc78..47e6bf07b 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,5 +1,5 @@ (ns core.manage - "Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" + ;"Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" (:use core.fdb)) (def __ALL-DBS__ (atom {})) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 92c588a41..075031896 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -17,38 +17,71 @@ (add-attr (make-attr :patient/tests #{} :db/ref :indexed true :cardinality :db/multiple)) (add-attr (make-attr :patient/symptoms (set symptoms) :string :cardinality :db/multiple)))) -(defn make-test [t-id tests-map types indexedPred] +(defn make-test [t-id tests-map types ] (let [ent (make-entity t-id)] (reduce #(add-attr %1 (make-attr (first %2) ;attr-name (second %2) ; attr value (get types (first %2) :number) ; attr type - :indexed (indexedPred (first %2)) ));indexed + :indexed true ));indexed ent tests-map))) (defn add-patient [id address symptoms] (transact hospital-db (add-entity (make-patient id address symptoms)))) (defn add-test-results-to-patient [pat-id test-result] - (let [test-id (:id test-result) ] - (transact hospital-db (add-entity test-result)) - (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) + (let [test-id (:id test-result) + a (transact hospital-db (add-entity test-result)) + ] + (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) ;; world setup (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) + + +;@hospital-db (add-patient :pat1 "London" ["fever" "cough"] ) + + + (add-patient :pat2 "London" ["fever" "cough"] ) -@hospital-db -(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 120 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} (fn [a] true))) -(add-test-results-to-patient :pat2 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} (fn [a] true))) +;@hospital-db +(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} )) +(add-test-results-to-patient :pat2 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} )) + ;(get-in @hospital-db [:timestamped 5 :AVET]) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) +(transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) +;(ind-at @hospital-db :EAVT) + ; (transact hospital-db (remove-entity :t2-pat1)) + ;(ind-at @hospital-db :EAVT) + + + (defn keep-on-equals [a b](if (= a b) a nil )) + +(def qq(map #(:db/variable (meta %)) (q-clauses [[ ?e :test/bp-systolic (> 150 ?b)][ ?e :test/bp-diastolic ?k]] ))) + +(defn keep-if-equals [s1 s2] + (map #(when (= %1 %2) %1) s1 s2) + ) + + +(first (keep-indexed #(when (variable? %2) %1) (reduce keep-if-equals-streams qq) )) + +(defn index-of-chaining-variable + [query-clauses] + (let [metas-seq (map #(:db/variable (meta %)) query-clauses) + collapse-seqs (fn [s1 s2] (map #(when (= %1 %2) %1) s1 s2)) + collapsed (reduce collapse-seqs metas-seq)] + (first (keep-indexed #(when (variable? %2) %1) collapsed)))) + +(index-of-chaining-variable (q-clauses [[ ?e :test/bp-systolic (> 150 ?b)][ ?e :test/bp-diastolic ?k]] )) + +(q @hospital-db {:find [?e ?k] :where [[ ?e :test/bp-systolic (> 200 ?b)] [ ?e :test/bp-diastolic ?k]]} ) + -;(transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) - ;(transact hospital-db (remove-entity :t2-pat1)) -(q @hospital-db {:where [[ ?e :test/bp-systolic (> 150 ?b)] [ ?e :test/bp-diastolic ?k]]} ) -;; (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) -;; (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) +(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) +(evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) From 59ebc7904ddbd74d3c744c30b2844ee8e901d446 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 7 Apr 2014 12:55:41 +0300 Subject: [PATCH 074/102] fixing the query answer mechanism to include full answers. Changing private functions to public ones towards structural refactoring --- functionalDB/src/core/fdb.clj | 100 ++++++++++++++++++++-------------- 1 file changed, 59 insertions(+), 41 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index f30680789..7b994f819 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -13,25 +13,24 @@ ;; (defrecord Database [timestamped top-id curr-time]) (defrecord Index [db-from-eav db-to-eav db-usage-pred db-leaf-index]) -(defrecord Timestamped [storage VAET AVET EAVT]) +(defrecord Timestamped [storage VAET AVET VEAT EAVT]) (defrecord Entity [id attrs]) (defrecord Attr [name value ts prev-ts]) -(defn- single? [attr] (= :db/single (:cardinality (meta attr)))) +(defn single? [attr] (= :db/single (:cardinality (meta attr)))) -(defn- indexed? [attr] (:indexed (meta attr))) - -(defn- ref? [attr] (= :db/ref (:type (meta attr)))) +(defn ref? [attr] (= :db/ref (:type (meta attr)))) (defn make-db "Create an empty database" [] (atom (Database. [(Timestamped. (initial-storage) ; storage (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))0) ; AVET - for filtering + (Index. #(vector %3 %1 %2) #(vector %2 %3 %1) #(not (not %))1) ; VEAT - for filtering (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))2) )] ; EAVT - for filtering 0 0))) -(defn indices[] [:VAET :AVET :EAVT]) +(defn indices[] [:VAET :AVET :VEAT :EAVT]) (defn make-entity "creates an entity, if id is not supplied, a running id is assigned to the entity" @@ -56,9 +55,9 @@ (defn collify [x] (if (coll? x) x [x])) -(defn- next-ts [db] (inc (:curr-time db))) +(defn next-ts [db] (inc (:curr-time db))) -(defn- next-id +(defn next-id "returns a pair composed of the id to use for the given entity and the next free running id in the database" [db ent] (let [top-id (:top-id db) @@ -86,7 +85,7 @@ ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) -(defn- update-attr-value +(defn update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" [attr value operation] (cond @@ -102,12 +101,12 @@ (let [attr-id (keyword (:name attr))] (assoc-in ent [:attrs attr-id] attr))) -(defn- update-creation-ts +(defn update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent ts-val] (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) -(defn- add-entry-to-index +(defn add-entry-to-index [index path operation] (if (= operation :db/remove) index @@ -116,7 +115,7 @@ to-be-updated-set (get-in index update-path #{})] (assoc-in index update-path (conj to-be-updated-set update-value) )))) -(defn- update-attr-in-index +(defn update-attr-in-index [index ent-id attr-name target-val operation] (let [ colled-target-val (collify target-val) add-entry-fn (fn [indx vl] (add-entry-to-index indx ((:db-from-eav index) ent-id attr-name vl) operation)) @@ -146,11 +145,11 @@ (defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) -(defn- update-attr-modification-time +(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) -(defn- update-attr +(defn update-attr [attr new-val new-ts operation] {:pre [(if (single? attr) (contains? #{:db/reset-to :db/remove} operation) @@ -159,14 +158,14 @@ (update-attr-modification-time new-ts) (update-attr-value new-val operation))) -(defn- remove-path-values +(defn remove-path-values [old-vals target-val operation] (cond (= operation :db/add) [] ; nothing to remove (= operation :db/reset-to) old-vals ; removing all of the old values (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller -(defn- update-index +(defn update-index [ent-id old-attr target-val operation timestamped ind-name] (if-not ((get-in timestamped [ind-name :db-usage-pred]) old-attr) timestamped @@ -181,7 +180,7 @@ (defn update-entity [storage e-id new-attr] (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) -(defn- update-timestamped +(defn update-timestamped [timestamped ent-id old-attr updated-attr new-val operation] (let [storage (:storage timestamped) new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] @@ -220,7 +219,7 @@ paths (map #((:db-from-eav index) ent-id attr-name %) datom-vals)] (reduce remove-entry-from-index index paths)))) -(defn- remove-entity-from-index +(defn remove-entity-from-index [ent timestamped ind-name] (let [ent-id (:id ent) index (ind-name timestamped) @@ -283,8 +282,9 @@ (defn variable? "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" - [x] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF - (or (= x "_") (= (first x) \?))) + ([x] (variable? x true)) + ([x accept_] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF + (or (and accept_ (= x "_")) (= (first x) \?)))) (defn ind-at "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EAVT" @@ -312,7 +312,7 @@ (variable? (str (last clause-item))) `#(~(first clause-item) ~(second clause-item) %))) ; was something like (> 42 ?a) (defmacro q-clause - "The aim of this macro to build for a datalog clause (a vector with three elements describing EAV) a vector of predicates that would operate on + "Build from a datalog clause (a vector with three elements describing EAV) a vector of predicates that would operate on an index, and set for that vector's metadata to be the names of the variables that the user assiged for each item in the clause" [clause] (loop [[frst-itm# & rst-itm#] clause exprs# [] metas# [] ] @@ -321,31 +321,40 @@ (with-meta exprs# {:db/variable metas#})))) (defmacro q-clauses - "create a vector of queries to operate on indexes based on the given vector of clauses " + "create a vector of queries to operate on indices, based on the given vector of clauses " [clauses] (loop [[frst# & rst#] clauses preds-vecs# [] ] (if-not frst# preds-vecs# (recur rst# `(conj ~preds-vecs# (q-clause ~frst#))) ))) - (defn choose-index ;; TODO !!! at the moment not optimzing, just returning always the AVET index + (defn index-of-chaining-variable + "A chaining variable is the variable that is found on all of the query clauses" + [query-clauses] + (let [metas-seq (map #(:db/variable (meta %)) query-clauses) + collapse-seqs (fn [s1 s2] (map #(when (= %1 %2) %1) s1 s2)) + collapsed (reduce collapse-seqs metas-seq)] + (first (keep-indexed #(when (variable? %2 false) %1) collapsed)))) + + (defn choose-index "Upon receiving a database and query clauses, this function responsible to deduce on which index in the db it is best to perfom the query clauses, and then return a vector in which the first element is the decided index and the second element is a function that knows how to restore an EAV structure from that decided index path structure." [db query] - (ind-at db :AVET)) + (let [var-ind (index-of-chaining-variable query) + ind-to-use (case var-ind 0 :AVET 1 :VEAT 2 :EAVT)] + (ind-at db ind-to-use))) (defn filter-index "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: -- At index 0 : the key used at the first level of the path. -- At index 1 : the key used at the second level of the path -- At index 2 : the elements (e-ids or attribute names) that are found at the leave of the path." - [index path-preds] (for [ path-pred path-preds :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (:db-from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta [k1 l2map] index ; keys and values of the first level - :when (lvl1-prd k1) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate + :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate [k2 l3-set] l2map ; keys and values of the second level - :when (lvl2-prd k2) ; filtering to keep only the keys and vals of keys that passed the second level predicate + :when (try (lvl2-prd k2) (catch Exception e false)) ; filtering to keep only the keys and vals of keys that passed the second level predicate :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them (with-meta [k1 k2 res] (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables @@ -357,7 +366,7 @@ (map vec) ; make each collection (actually a set) into a vector (reduce into []) ;reduce all the vectors into one big vector (frequencies) ; count for each item in how many collections (sets) it was in originally - (filter #(= num-of-conditions (last %))) ; keep only the items that answered all of the conditions + (filter #(<= num-of-conditions (last %))) ; keep only the items that answered all of the conditions (map first) ; take from the duos the items themeselves (set))) ; return it as set @@ -373,17 +382,16 @@ [index path ] (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index - all-path (interleave seq-path meta-path)] + all-path (interleave meta-path seq-path)] (apply (partial map vector) all-path))) (defn merge-query-and-meta [q-res index] (let [seq-res-path (mapcat (partial seqify-result-path index) q-res ) - reversed-res-path (map reverse seq-res-path) - paired-path-meta (map (partial partition 2) reversed-res-path)] - (reduce #(assoc-in %1 (butlast %2) (last %2)) {} paired-path-meta))) + res-path (map #(->> %1 (partition 2)(apply (:db-to-eav index))) seq-res-path)] + (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) (defn query-index - "an index is a 3 level map (VAE, AVE, EVA) that is found at a specific time (T). + "an index is a 3 level map (AVE, EAV, VEA) that is found at a specific time (T). The path-preds argument is a seq of vectors. Each of these vectors contains two elements: -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, E -> an id matching predicator) -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. @@ -395,14 +403,24 @@ relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates (filter #(not-empty (last %)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items -(defn values-of-vars - "For a given query result (a merge of the result and the metadata), returns the variable values found at the requested variable set" +(defn resultify-bind-pair + "A bind pair is composed of two elements - the variable name and its value. Resultifying means to check whether the variable is suppose to be part of the + result, and if it does, adds it to the accumulated result" + [vars-set accum pair] + (let [[ var-name _] pair] + (if (contains? vars-set var-name) (conj accum pair) accum))) + +(defn resultify-av-pair + "An av pair is a pair composed of two binding pairs, one for an attribute and one for the attribute's value" + [vars-set accum-res av-pair] + (reduce (partial resultify-bind-pair vars-set) accum-res av-pair)) + +(defn vars-in-query-res + "this function would look for all the binding found in the query result and return the binding that were requested by the user (captured at the vars-set)" [vars-set q-res] - (let [[root-pair child-map] q-res ; breaking the result to have both the root pair and all the rest of the pairs - united-children (CS/union (set (keys child-map)) (set (vals child-map))) ; uniting the child pairs to one set of bindings - all-binding-set (conj united-children root-pair) ; adding to the set of bindings the root pair - relevant-binds (filter #(contains? vars-set (first %)) all-binding-set)] ; keeping only the relevant pairs - (zipmap (map first relevant-binds) (map second relevant-binds))) ) ; turning the pairs into map + (let [[e-pair av-map] q-res + e-res (resultify-bind-pair vars-set [] e-pair )] + (reduce (partial resultify-av-pair vars-set) e-res av-map))) (defmacro settify [coll] (set (map str coll))) @@ -415,7 +433,7 @@ q-res# (query-index ind# query#) merged-res# (merge-query-and-meta q-res# ind#) find-items# (settify ~(:find query)) - res# (map (partial values-of-vars (set find-items#) ) merged-res# ) + res# (map (partial vars-in-query-res find-items# ) merged-res#) ]res# )) From 1f84ca0ed2e01abc3c4303c841ef7721a7cc5932 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 7 Apr 2014 14:31:52 +0300 Subject: [PATCH 075/102] refactoring the query mechanisms out of the main module (towards it being an API module) --- functionalDB/src/core/fdb.clj | 240 +++++-------------------------- functionalDB/src/core/manage.clj | 2 +- functionalDB/src/core/query.clj | 150 +++++++++++++++++++ 3 files changed, 190 insertions(+), 202 deletions(-) create mode 100644 functionalDB/src/core/query.clj diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 7b994f819..eed2ffcd9 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,16 +1,7 @@ (ns core.fdb - [:use core.storage] + [:use [core storage query]] [:require [clojure.set :as CS :only (union difference intersection)]]) -;; -- storage enty-id -> entity {:attrs -> {attr-name -> attr {:value -> the-value}}} -;; -- VAET structed like this: {REFed-ent-id -> {attrName -> #{REFing-elems-ids}}} -;; this basically provides this info: for each entity that is REFFed by others (V), separated by the names of the attribute used for reffing (A), hold the set of the ids of the REFing (E), all this at a given time (T) -;; can be used to know who REFs a specific entity -;; -- AVET structed like this: {attrName-> {attr-val -> #{holding-elems-ids}}} -;; this basically provides this info: for each attributeName (A) that we chose to index, separated by the values of the attribute (V), hold the set of the ids of the hold thes attributes (E), all this at a given time (T) -;; can be used to know who are the entities that have a specific attribute with a specific value -;; -- EAVT structed like this: {ent-id -> {attr-name -> #{attr-vals}}} mimics the storage structure -;; (defrecord Database [timestamped top-id curr-time]) (defrecord Index [db-from-eav db-to-eav db-usage-pred db-leaf-index]) (defrecord Timestamped [storage VAET AVET VEAT EAVT]) @@ -21,7 +12,9 @@ (defn ref? [attr] (= :db/ref (:type (meta attr)))) -(defn make-db "Create an empty database" [] +(defn make-db + "Create an empty database" + [] (atom (Database. [(Timestamped. (initial-storage) ; storage (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins @@ -130,7 +123,8 @@ add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) -(defn fix-new-entity [db ent] +(defn fix-new-entity + [db ent] (let [[ent-id next-top-id] (next-id db ent) new-ts (next-ts db)] [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) @@ -165,6 +159,27 @@ (= operation :db/reset-to) old-vals ; removing all of the old values (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller +(defn remove-entry-from-index + [index path] + (let [path-head (first path) + path-to-items (butlast path) + val-to-remove (last path) + old-entries-set (get-in index path-to-items)] + (cond + (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here + (and (= 1 (count old-entries-set) ) (= 1 (count (path-head index)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely + (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it + :else (update-in index path-to-items disj val-to-remove)))) + +(defn remove-entries-from-index + [ent-id operation index attr] + (if (= operation :db/add) + index + (let [attr-name (:name attr) + datom-vals (collify (:value attr)) + paths (map #((:db-from-eav index) ent-id attr-name %) datom-vals)] + (reduce remove-entry-from-index index paths)))) + (defn update-index [ent-id old-attr target-val operation timestamped ind-name] (if-not ((get-in timestamped [ind-name :db-usage-pred]) old-attr) @@ -177,7 +192,8 @@ updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] (assoc timestamped ind-name updated-index)))) -(defn update-entity [storage e-id new-attr] +(defn update-entity + [storage e-id new-attr] (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) (defn update-timestamped @@ -198,27 +214,6 @@ new-db (update-in db [:timestamped] conj fully-updated-timestamped)] new-db))) -(defn remove-entry-from-index - [index path] - (let [path-head (first path) - path-to-items (butlast path) - val-to-remove (last path) - old-entries-set (get-in index path-to-items)] - (cond - (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here - (and (= 1 (count old-entries-set) ) (= 1 (count (path-head index)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely - (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it - :else (update-in index path-to-items disj val-to-remove)))) - -(defn remove-entries-from-index - [ent-id operation index attr] - (if (= operation :db/add) - index - (let [attr-name (:name attr) - datom-vals (collify (:value attr)) - paths (map #((:db-from-eav index) ent-id attr-name %) datom-vals)] - (reduce remove-entry-from-index index paths)))) - (defn remove-entity-from-index [ent timestamped ind-name] (let [ent-id (:id ent) @@ -228,13 +223,15 @@ remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) -(defn reffing-datoms-to[e-id timestamped] +(defn reffing-datoms-to + [e-id timestamped] (let [vaet (:VAET timestamped)] (for [[attr-name reffing-set] (e-id vaet) reffing reffing-set] [reffing attr-name e-id]))) -(defn remove-back-refs [db e-id timestamped] +(defn remove-back-refs + [db e-id timestamped] (let [refing-datoms (reffing-datoms-to e-id timestamped) remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) clean-db (reduce remove-fn db refing-datoms)] @@ -278,164 +275,16 @@ (defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) -;;;;;;;;;;;;;;;;;;;;;;; queries - -(defn variable? - "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" - ([x] (variable? x true)) - ([x accept_] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF - (or (and accept_ (= x "_")) (= (first x) \?)))) - -(defn ind-at - "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EAVT" - ([db kind] - (ind-at db kind (:curr-time db))) - ([db kind ts] - (kind ((:timestamped db) ts)))) - -(defmacro clause-item-meta - "Finds the name of the variable at an item of a datalog clause element. If no variable, returning nil" - [clause-item] - (cond - (coll? clause-item) (first (filter variable? (map str clause-item))) ; the item is an s-expression, need to treat it as a coll, by going over it and returning the name of the variable - (variable? (str clause-item)) (str clause-item) ; the item is a simple variable - :no-variable-in-clause nil)) ; the item is a value and not a variable - -(defmacro clause-item-expr - "Create a predicate for each element in the datalog clause" - [clause-item] - (cond - (variable? (str clause-item)) #(= % %) ; simple one, was something like ?a - (not (coll? clause-item)) `#(= % ~(identity clause-item)) ; simple value given, was something like :likes - (= 2 (count clause-item)) `#(~(first clause-item) %) ; was something like (pos? ?a) - (variable? (str (second clause-item))) `#(~(first clause-item) % ~(last clause-item)) ; was something like (> ?a 42) - (variable? (str (last clause-item))) `#(~(first clause-item) ~(second clause-item) %))) ; was something like (> 42 ?a) - -(defmacro q-clause - "Build from a datalog clause (a vector with three elements describing EAV) a vector of predicates that would operate on - an index, and set for that vector's metadata to be the names of the variables that the user assiged for each item in the clause" - [clause] - (loop [[frst-itm# & rst-itm#] clause exprs# [] metas# [] ] - (if frst-itm# - (recur rst-itm# (conj exprs# `(clause-item-expr ~ frst-itm#)) (conj metas#`(clause-item-meta ~ frst-itm#))) - (with-meta exprs# {:db/variable metas#})))) - - (defmacro q-clauses - "create a vector of queries to operate on indices, based on the given vector of clauses " - [clauses] - (loop [[frst# & rst#] clauses preds-vecs# [] ] - (if-not frst# preds-vecs# - (recur rst# `(conj ~preds-vecs# (q-clause ~frst#))) ))) - - (defn index-of-chaining-variable - "A chaining variable is the variable that is found on all of the query clauses" - [query-clauses] - (let [metas-seq (map #(:db/variable (meta %)) query-clauses) - collapse-seqs (fn [s1 s2] (map #(when (= %1 %2) %1) s1 s2)) - collapsed (reduce collapse-seqs metas-seq)] - (first (keep-indexed #(when (variable? %2 false) %1) collapsed)))) - - (defn choose-index - "Upon receiving a database and query clauses, this function responsible to deduce on which index in the db it is best to perfom the query clauses, and then return - a vector in which the first element is the decided index and the second element is a function that knows how to restore an EAV structure from that decided index path structure." - [db query] - (let [var-ind (index-of-chaining-variable query) - ind-to-use (case var-ind 0 :AVET 1 :VEAT 2 :EAVT)] - (ind-at db ind-to-use))) - -(defn filter-index - "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: - -- At index 0 : the key used at the first level of the path. - -- At index 1 : the key used at the second level of the path - -- At index 2 : the elements (e-ids or attribute names) that are found at the leave of the path." - [index path-preds] - (for [ path-pred path-preds - :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (:db-from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta - [k1 l2map] index ; keys and values of the first level - :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate - [k2 l3-set] l2map ; keys and values of the second level - :when (try (lvl2-prd k2) (catch Exception e false)) ; filtering to keep only the keys and vals of keys that passed the second level predicate - :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them - (with-meta [k1 k2 res] (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables - - (defn items-that-answer-all-conditions - "takes the sequence of all the items collection, each such collection answered one condition, we test here what are the items that answered all of the conditions - i.e., what items are found at exactly 'num-of-conditions' of such collections " - [items-seq num-of-conditions] - (->> items-seq ; take the items-seq - (map vec) ; make each collection (actually a set) into a vector - (reduce into []) ;reduce all the vectors into one big vector - (frequencies) ; count for each item in how many collections (sets) it was in originally - (filter #(<= num-of-conditions (last %))) ; keep only the items that answered all of the conditions - (map first) ; take from the duos the items themeselves - (set))) ; return it as set - -(defn mask-path-leaf-with-items - "a path is a vector triplet, at the first position there's a set of the path's items, here we intersect the path's items with the set of relevant items" - [index relevant-items path] - (update-in path [2] CS/intersection relevant-items ));) - - (defn seqify-result-path - "A result-path is a path whose leaves are the items representing the items of the query chaining variable. This function - returns for a result path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item is followed - by it's variable name as was inserted in the query" - [index path ] - (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] - meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index - all-path (interleave meta-path seq-path)] - (apply (partial map vector) all-path))) - -(defn merge-query-and-meta [q-res index] - (let [seq-res-path (mapcat (partial seqify-result-path index) q-res ) - res-path (map #(->> %1 (partition 2)(apply (:db-to-eav index))) seq-res-path)] - (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) - -(defn query-index - "an index is a 3 level map (AVE, EAV, VEA) that is found at a specific time (T). - The path-preds argument is a seq of vectors. Each of these vectors contains two elements: - -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, E -> an id matching predicator) - -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. - -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] - The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" - [index path-preds] - (let [filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path - relevant-items (items-that-answer-all-conditions (map last filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths - relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates - (filter #(not-empty (last %)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items - -(defn resultify-bind-pair - "A bind pair is composed of two elements - the variable name and its value. Resultifying means to check whether the variable is suppose to be part of the - result, and if it does, adds it to the accumulated result" - [vars-set accum pair] - (let [[ var-name _] pair] - (if (contains? vars-set var-name) (conj accum pair) accum))) - -(defn resultify-av-pair - "An av pair is a pair composed of two binding pairs, one for an attribute and one for the attribute's value" - [vars-set accum-res av-pair] - (reduce (partial resultify-bind-pair vars-set) accum-res av-pair)) - -(defn vars-in-query-res - "this function would look for all the binding found in the query result and return the binding that were requested by the user (captured at the vars-set)" - [vars-set q-res] - (let [[e-pair av-map] q-res - e-res (resultify-bind-pair vars-set [] e-pair )] - (reduce (partial resultify-av-pair vars-set) e-res av-map))) - -(defmacro settify [coll] (set (map str coll))) - (defmacro q "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). At the moment support only filtering queries, no joins is also assumed." [db query] - `(let [query# (q-clauses ~(:where query) ) - ind# (choose-index ~db query#) - q-res# (query-index ind# query#) - merged-res# (merge-query-and-meta q-res# ind#) - find-items# (settify ~(:find query)) - res# (map (partial vars-in-query-res find-items# ) merged-res#) - ]res# - )) + `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query + ind# (choose-index ~db query#) ; selecting which index to use + q-res# (query-index ind# query#) ; actually quering, from now on just preparing results + merged-res# (merge-query-and-meta q-res# ind#) ; building a meta + real results structure + needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables + (map (partial locate-vars-in-query-res needed-vars# ) merged-res#))) ; picking the needed variables from the query result (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" @@ -450,14 +299,3 @@ [db ts] (let [timestamped-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped timestamped-before :curr-time ts))) - -;; (defn ref-to-as -;; "returns a seq of all the entities that have REFed to the give entity with the given attr-name (alternativly had an attribute -;; named attr-name whose type is :db/ref and the value was ent-id), all this at a given time" -;; ([db ent-id attr-name] (ref-to-as db ent-id attr-name (:curr-time db))) -;; ([db ent-id attr-name ts] -;; (let [indices ((:timestamped db) ts) -;; reffing-ids (get-in indices [:VAET ent-id attr-name])] -;; (map #(get-in indices [:EAVT %]) reffing-ids )))) - - diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index 47e6bf07b..a58ebfc78 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,5 +1,5 @@ (ns core.manage - ;"Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" + "Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" (:use core.fdb)) (def __ALL-DBS__ (atom {})) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj new file mode 100644 index 000000000..3e7078c29 --- /dev/null +++ b/functionalDB/src/core/query.clj @@ -0,0 +1,150 @@ +(ns core.query + [:require [clojure.set :as CS :only (intersection)]]) + +(defn variable? + "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" + ([x] (variable? x true)) + ([x accept_] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF + (or (and accept_ (= x "_")) (= (first x) \?)))) + +(defn ind-at + "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EAVT" + ([db kind] + (ind-at db kind (:curr-time db))) + ([db kind ts] + (kind ((:timestamped db) ts)))) + +(defmacro clause-item-meta + "Finds the name of the variable at an item of a datalog clause element. If no variable, returning nil" + [clause-item] + (cond + (coll? clause-item) (first (filter variable? (map str clause-item))) ; the item is an s-expression, need to treat it as a coll, by going over it and returning the name of the variable + (variable? (str clause-item)) (str clause-item) ; the item is a simple variable + :no-variable-in-clause nil)) ; the item is a value and not a variable + +(defmacro clause-item-expr + "Create a predicate for each element in the datalog clause" + [clause-item] + (cond + (variable? (str clause-item)) #(= % %) ; simple one, was something like ?a + (not (coll? clause-item)) `#(= % ~(identity clause-item)) ; simple value given, was something like :likes + (= 2 (count clause-item)) `#(~(first clause-item) %) ; was something like (pos? ?a) + (variable? (str (second clause-item))) `#(~(first clause-item) % ~(last clause-item)) ; was something like (> ?a 42) + (variable? (str (last clause-item))) `#(~(first clause-item) ~(second clause-item) %))) ; was something like (> 42 ?a) + +(defmacro q-clause + "Build from a datalog clause (a vector with three elements describing EAV) a vector of predicates that would operate on + an index, and set for that vector's metadata to be the names of the variables that the user assiged for each item in the clause" + [clause] + (loop [[frst-itm# & rst-itm#] clause exprs# [] metas# [] ] + (if frst-itm# + (recur rst-itm# (conj exprs# `(clause-item-expr ~ frst-itm#)) (conj metas#`(clause-item-meta ~ frst-itm#))) + (with-meta exprs# {:db/variable metas#})))) + + (defmacro q-clauses + "create a vector of queries to operate on indices, based on the given vector of clauses " + [clauses] + (loop [[frst# & rst#] clauses preds-vecs# [] ] + (if-not frst# preds-vecs# + (recur rst# `(conj ~preds-vecs# (q-clause ~frst#))) ))) + + (defn index-of-chaining-variable + "A chaining variable is the variable that is found on all of the query clauses" + [query-clauses] + (let [metas-seq (map #(:db/variable (meta %)) query-clauses) + collapse-seqs (fn [s1 s2] (map #(when (= %1 %2) %1) s1 s2)) + collapsed (reduce collapse-seqs metas-seq)] + (first (keep-indexed #(when (variable? %2 false) %1) collapsed)))) + + (defn choose-index + "Upon receiving a database and query clauses, this function responsible to deduce on which index in the db it is best to perfom the query clauses, and then return + a vector in which the first element is the decided index and the second element is a function that knows how to restore an EAV structure from that decided index path structure." + [db query] + (let [var-ind (index-of-chaining-variable query) + ind-to-use (case var-ind 0 :AVET 1 :VEAT 2 :EAVT)] + (ind-at db ind-to-use))) + +(defn filter-index + "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: + -- At index 0 : the key used at the first level of the path. + -- At index 1 : the key used at the second level of the path + -- At index 2 : the elements (e-ids or attribute names) that are found at the leave of the path." + [index path-preds] + (for [ path-pred path-preds + :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (:db-from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta + [k1 l2map] index ; keys and values of the first level + :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate + [k2 l3-set] l2map ; keys and values of the second level + :when (try (lvl2-prd k2) (catch Exception e false)) ; filtering to keep only the keys and vals of keys that passed the second level predicate + :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them + (with-meta [k1 k2 res] (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables + + (defn items-that-answer-all-conditions + "takes the sequence of all the items collection, each such collection answered one condition, we test here what are the items that answered all of the conditions + i.e., what items are found at exactly 'num-of-conditions' of such collections " + [items-seq num-of-conditions] + (->> items-seq ; take the items-seq + (map vec) ; make each collection (actually a set) into a vector + (reduce into []) ;reduce all the vectors into one big vector + (frequencies) ; count for each item in how many collections (sets) it was in originally + (filter #(<= num-of-conditions (last %))) ; keep only the items that answered all of the conditions + (map first) ; take from the duos the items themeselves + (set))) ; return it as set + +(defn mask-path-leaf-with-items + "a path is a vector triplet, at the first position there's a set of the path's items, here we intersect the path's items with the set of relevant items" + [index relevant-items path] + (update-in path [2] CS/intersection relevant-items ));) + + (defn seqify-result-path + "A result-path is a path whose leaves are the items representing the items of the query chaining variable. This function + returns for a result path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item is followed + by it's variable name as was inserted in the query" + [index path] + (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] + meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index + all-path (interleave meta-path seq-path)] + (apply (partial map vector) all-path))) + +(defn merge-query-and-meta + "A function that receives the query results and restructues them to be a map, where + the key is a binding pair of a found entity-id, and the value is also a map, where its key is the binding pair of a found + attribute, and the value is the binding pair of that found attribute's value" + [q-res index] + (let [seq-res-path (mapcat (partial seqify-result-path index) q-res) ; seq-ing a result to hold the meta + res-path (map #(->> %1 (partition 2)(apply (:db-to-eav index))) seq-res-path)] ; making binding pairs + (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) ; structuring the pairs into the wanted map structure + +(defn query-index + "an index is a 3 level map (AVE, EAV, VEA) that is found at a specific time (T). + The path-preds argument is a seq of vectors. Each of these vectors contains two elements: + -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, E -> an id matching predicator) + -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. + -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] + The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" + [index path-preds] + (let [filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path + relevant-items (items-that-answer-all-conditions (map last filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths + relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates + (filter #(not-empty (last %)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items + +(defn resultify-bind-pair + "A bind pair is composed of two elements - the variable name and its value. Resultifying means to check whether the variable is suppose to be part of the + result, and if it does, adds it to the accumulated result" + [vars-set accum pair] + (let [[ var-name _] pair] + (if (contains? vars-set var-name) (conj accum pair) accum))) + +(defn resultify-av-pair + "An av pair is a pair composed of two binding pairs, one for an attribute and one for the attribute's value" + [vars-set accum-res av-pair] + (reduce (partial resultify-bind-pair vars-set) accum-res av-pair)) + +(defn locate-vars-in-query-res + "this function would look for all the binding found in the query result and return the binding that were requested by the user (captured at the vars-set)" + [vars-set q-res] + (let [[e-pair av-map] q-res + e-res (resultify-bind-pair vars-set [] e-pair )] + (reduce (partial resultify-av-pair vars-set) e-res av-map))) + +(defmacro settify [coll] (set (map str coll))) From 521ffe960f123c5d70d993de5e3ce170206070f1 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 7 Apr 2014 16:22:22 +0300 Subject: [PATCH 076/102] refactoring to hold the DB constructs in a seperate module --- functionalDB/src/core/constructs.clj | 58 +++++++++++++++++++++++++++ functionalDB/src/core/fdb.clj | 60 ++++------------------------ functionalDB/src/core/manage.clj | 2 +- functionalDB/src/core/query.clj | 9 +++-- 4 files changed, 72 insertions(+), 57 deletions(-) create mode 100644 functionalDB/src/core/constructs.clj diff --git a/functionalDB/src/core/constructs.clj b/functionalDB/src/core/constructs.clj new file mode 100644 index 000000000..d3d76dafa --- /dev/null +++ b/functionalDB/src/core/constructs.clj @@ -0,0 +1,58 @@ +(ns core.constructs + (:use core.storage)) + +(defrecord Database [timestamped top-id curr-time]) +(defrecord Timestamped [storage VAET AVET VEAT EAVT]) +(defrecord Entity [id attrs]) +(defrecord Attr [name value ts prev-ts]) + +(defn make-index + "An index is a tree, implemented by nested maps, each level corresponds to either entity-id, attribute name or a value, where the leaves of the tree are sets. + The order of the levels changes from one index to another, to allow different quering on different indices. It is possible to reorder a path in the tree to an EAV structure + using the to-eav function, or transform an EAV to a specific index path using the from-eav, both function reside in the metadataa of the index. The leaf index is the index of + the leaf element in an EAV triplet. The usage-pred is a predicate to decide whether to index a specific attribute in an entity or not." + [from-eav to-eav usage-pred leaf-index] + (with-meta {} {:from-eav from-eav :to-eav to-eav :usage-pred usage-pred :leaf-index leaf-index})) + +(defn from-eav [index] (:from-eav (meta index))) +(defn to-eav [index] (:to-eav (meta index))) +(defn usage-pred [index] (:usage-pred (meta index))) +(defn leaf-index [index] (:usage-pred (meta leaf-index))) + +(defn single? [attr] (= :db/single (:cardinality (meta attr)))) + +(defn ref? [attr] (= :db/ref (:type (meta attr)))) + +(defn make-db + "Create an empty database" + [] + (atom (Database. [(Timestamped. + (initial-storage) ; storage + (make-index #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins + (make-index #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %)) 0) ; AVET - for filtering + (make-index #(vector %3 %1 %2) #(vector %2 %3 %1) #(not (not %)) 1) ; VEAT - for filtering + (make-index #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %)) 2) )] ; EAVT - for filtering + 0 0))) + +(defn indices[] [:VAET :AVET :VEAT :EAVT]) + +(defn make-entity + "creates an entity, if id is not supplied, a running id is assigned to the entity" + ([] (make-entity :db/no-id-yet )) + ([id] (Entity. id {}))) + +(defn make-attr + "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguguments. + The type of the attribute may be either :string, :number, :boolean or :db/ref . If the type is :db/ref, the value is an id of another entity and indexing of backpointing is maintained. + The named arguments are as follows: + :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. + :cardinality - the cardinality of an attribute, can be either: + :db/single - which means that this attribute can be a single value at any given time (this is the default cardinality) + :db/multiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): + :db/add - adds a set of values to the currently existing set of values + :db/reset-to - resets the value of this attribute to be the given set of values + :db/remove - removes the given set of values from the attribute's current set of values" + ([name value type ; these ones are required + & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] + {:pre [(contains? #{true false} indexed) (contains? #{:db/single :db/multiple} cardinality)]} + (with-meta (Attr. name value -1 -1) {:type type :indexed indexed :cardinality cardinality} ))) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index eed2ffcd9..d973d084c 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,51 +1,7 @@ (ns core.fdb - [:use [core storage query]] + [:use [core storage query constructs]] [:require [clojure.set :as CS :only (union difference intersection)]]) -(defrecord Database [timestamped top-id curr-time]) -(defrecord Index [db-from-eav db-to-eav db-usage-pred db-leaf-index]) -(defrecord Timestamped [storage VAET AVET VEAT EAVT]) -(defrecord Entity [id attrs]) -(defrecord Attr [name value ts prev-ts]) - -(defn single? [attr] (= :db/single (:cardinality (meta attr)))) - -(defn ref? [attr] (= :db/ref (:type (meta attr)))) - -(defn make-db - "Create an empty database" - [] - (atom (Database. [(Timestamped. - (initial-storage) ; storage - (Index. #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins - (Index. #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %))0) ; AVET - for filtering - (Index. #(vector %3 %1 %2) #(vector %2 %3 %1) #(not (not %))1) ; VEAT - for filtering - (Index. #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %))2) )] ; EAVT - for filtering - 0 0))) - -(defn indices[] [:VAET :AVET :VEAT :EAVT]) - -(defn make-entity - "creates an entity, if id is not supplied, a running id is assigned to the entity" - ([] (make-entity :db/no-id-yet )) - ([id] (Entity. id {}))) - -(defn make-attr - "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguguments. - The type of the attribute may be either :string, :number, :boolean or :db/ref . If the type is :db/ref, the value is an id of another entity and indexing of backpointing is maintained. - The named arguments are as follows: - :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. - :cardinality - the cardinality of an attribute, can be either: - :db/single - which means that this attribute can be a single value at any given time (this is the default cardinality) - :db/multiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): - :db/add - adds a set of values to the currently existing set of values - :db/reset-to - resets the value of this attribute to be the given set of values - :db/remove - removes the given set of values from the attribute's current set of values" - ([name value type ; these ones are required - & {:keys [indexed cardinality] :or {indexed false cardinality :db/single}} ] - {:pre [(contains? #{true false} indexed) (contains? #{:db/single :db/multiple} cardinality)]} - (with-meta (Attr. name value -1 -1) {:type type :indexed indexed :cardinality cardinality} ))) - (defn collify [x] (if (coll? x) x [x])) (defn next-ts [db] (inc (:curr-time db))) @@ -111,15 +67,15 @@ (defn update-attr-in-index [index ent-id attr-name target-val operation] (let [ colled-target-val (collify target-val) - add-entry-fn (fn [indx vl] (add-entry-to-index indx ((:db-from-eav index) ent-id attr-name vl) operation)) - ] (reduce add-entry-fn index colled-target-val))) + add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] + (reduce add-entry-fn index colled-target-val))) (defn add-entity-to-index [ent timestamped ind-name] (let [ent-id (:id ent) index (ind-name timestamped) all-attrs (vals (:attrs ent)) - relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) + relevant-attrs (filter #((usage-pred index) %) all-attrs ) add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) @@ -167,7 +123,7 @@ old-entries-set (get-in index path-to-items)] (cond (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here - (and (= 1 (count old-entries-set) ) (= 1 (count (path-head index)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely + (and (= 1 (count old-entries-set) ) (= 1 (count (index path-head)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it :else (update-in index path-to-items disj val-to-remove)))) @@ -177,12 +133,12 @@ index (let [attr-name (:name attr) datom-vals (collify (:value attr)) - paths (map #((:db-from-eav index) ent-id attr-name %) datom-vals)] + paths (map #((from-eav index) ent-id attr-name %) datom-vals)] (reduce remove-entry-from-index index paths)))) (defn update-index [ent-id old-attr target-val operation timestamped ind-name] - (if-not ((get-in timestamped [ind-name :db-usage-pred]) old-attr) + (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) timestamped (let [index (ind-name timestamped) old-vals (:value old-attr) @@ -219,7 +175,7 @@ (let [ent-id (:id ent) index (ind-name timestamped) all-attrs (vals (:attrs ent)) - relevant-attrs (filter #((:db-usage-pred index) %) all-attrs ) + relevant-attrs (filter #((usage-pred index) %) all-attrs ) remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index a58ebfc78..0aacd7f54 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,6 +1,6 @@ (ns core.manage "Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" - (:use core.fdb)) + (:use core.constructs)) (def __ALL-DBS__ (atom {})) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index 3e7078c29..845ce3d37 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -1,5 +1,6 @@ (ns core.query - [:require [clojure.set :as CS :only (intersection)]]) + [:use [core constructs] + [clojure.set :as CS :only (intersection)]]) (defn variable? "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" @@ -71,7 +72,7 @@ -- At index 2 : the elements (e-ids or attribute names) that are found at the leave of the path." [index path-preds] (for [ path-pred path-preds - :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (:db-from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta + :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta [k1 l2map] index ; keys and values of the first level :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate [k2 l3-set] l2map ; keys and values of the second level @@ -102,7 +103,7 @@ by it's variable name as was inserted in the query" [index path] (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] - meta-path(apply (:db-from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index + meta-path(apply (from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index all-path (interleave meta-path seq-path)] (apply (partial map vector) all-path))) @@ -112,7 +113,7 @@ attribute, and the value is the binding pair of that found attribute's value" [q-res index] (let [seq-res-path (mapcat (partial seqify-result-path index) q-res) ; seq-ing a result to hold the meta - res-path (map #(->> %1 (partition 2)(apply (:db-to-eav index))) seq-res-path)] ; making binding pairs + res-path (map #(->> %1 (partition 2)(apply (to-eav index))) seq-res-path)] ; making binding pairs (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) ; structuring the pairs into the wanted map structure (defn query-index From 2cd6539fe626984ccbb0989cb87d9690701bbf3a Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 7 Apr 2014 17:20:19 +0300 Subject: [PATCH 077/102] cleanup --- functionalDB/src/core/fdb.clj | 30 ++++++++--------- functionalDB/src/scenarios/hospital.clj | 44 +++++++++---------------- 2 files changed, 29 insertions(+), 45 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index d973d084c..17e288f1f 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -143,9 +143,9 @@ (let [index (ind-name timestamped) old-vals (:value old-attr) attr-name (:name old-attr) - remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index ent-id operation index old-attr) - updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] + remove-paths (remove-path-values old-vals target-val operation) + cleaned-index (remove-entries-from-index ent-id operation index old-attr) + updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] (assoc timestamped ind-name updated-index)))) (defn update-entity @@ -166,9 +166,8 @@ timestamped (last (:timestamped db)) attr (attr-at db ent-id attr-name) updated-attr (update-attr attr new-val update-ts operation) - fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation) - new-db (update-in db [:timestamped] conj fully-updated-timestamped)] - new-db))) + fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] + (update-in db [:timestamped] conj fully-updated-timestamped)))) (defn remove-entity-from-index [ent timestamped ind-name] @@ -183,7 +182,7 @@ [e-id timestamped] (let [vaet (:VAET timestamped)] (for [[attr-name reffing-set] (e-id vaet) - reffing reffing-set] + reffing reffing-set] [reffing attr-name e-id]))) (defn remove-back-refs @@ -196,11 +195,10 @@ (defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) - timestamped (remove-back-refs db ent-id (last (:timestamped db))) - timestamped (update-in timestamped [:VAET] dissoc ent-id) - new-timestamped (assoc timestamped :storage (remove-entity-from-storage (:storage timestamped) ent)) - remove-fn (partial remove-entity-from-index ent) - new-timestamped (reduce remove-fn new-timestamped (indices))] + timestamped (remove-back-refs db ent-id (last (:timestamped db))) + retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) + no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) + new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) (defn transact-on-db @@ -209,15 +207,13 @@ (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [initial-timestamped (:timestamped initial-db) - new-timestamped (last (:timestamped transacted)) - res (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) - :curr-time (next-ts initial-db) :top-id (:top-id transacted))] - res)))) + new-timestamped (last (:timestamped transacted))] + (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) (defmacro _transact [db op & txs] (when txs - (loop [[frst-tx# & rst-tx#] txs res# [op db 'transact-on-db] accum-txs# []] + (loop [[frst-tx# & rst-tx#] txs res# [op db `transact-on-db] accum-txs# []] (if frst-tx# (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) (list* (conj res# accum-txs#)))))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 075031896..c0083cdbf 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,5 +1,5 @@ (ns scenarios.hospital - (:use core.fdb) + (:use [core fdb query constructs] ) [:require [core.manage :as M] [clojure.set :as CS :only (union difference )]]) @@ -37,50 +37,38 @@ ;; world setup (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) - -;@hospital-db (add-patient :pat1 "London" ["fever" "cough"] ) - - (add-patient :pat2 "London" ["fever" "cough"] ) -;@hospital-db + (add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} )) (add-test-results-to-patient :pat2 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} )) - ;(get-in @hospital-db [:timestamped 5 :AVET]) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) (transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) -;(ind-at @hospital-db :EAVT) ; (transact hospital-db (remove-entity :t2-pat1)) - ;(ind-at @hospital-db :EAVT) - (defn keep-on-equals [a b](if (= a b) a nil )) -(def qq(map #(:db/variable (meta %)) (q-clauses [[ ?e :test/bp-systolic (> 150 ?b)][ ?e :test/bp-diastolic ?k]] ))) - -(defn keep-if-equals [s1 s2] - (map #(when (= %1 %2) %1) s1 s2) - ) - - -(first (keep-indexed #(when (variable? %2) %1) (reduce keep-if-equals-streams qq) )) - -(defn index-of-chaining-variable - [query-clauses] - (let [metas-seq (map #(:db/variable (meta %)) query-clauses) - collapse-seqs (fn [s1 s2] (map #(when (= %1 %2) %1) s1 s2)) - collapsed (reduce collapse-seqs metas-seq)] - (first (keep-indexed #(when (variable? %2) %1) collapsed)))) - -(index-of-chaining-variable (q-clauses [[ ?e :test/bp-systolic (> 150 ?b)][ ?e :test/bp-diastolic ?k]] )) - +;(def qq(map #(:db/variable (meta %)) (q-clauses [[ ?e :test/bp-systolic (> 150 ?b)][ ?e :test/bp-diastolic ?k]] ))) +(ind-at @hospital-db :EAVT) (q @hospital-db {:find [?e ?k] :where [[ ?e :test/bp-systolic (> 200 ?b)] [ ?e :test/bp-diastolic ?k]]} ) +;(def ind (ind-at @hospital-db :VEAT)) +;(def q-m + (q @hospital-db {:find [?a ?b] :where [[ _ ?a (> 200 ?b)] ]}) + (q @hospital-db {:find [?e ?k ] :where [[ ?e :test/bp-systolic (> 180 ?b)][ ?e :test/bp-diastolic ?k] ]}) + ;) + ;(map (partial vars-in-query-res st ) q-m) +;(vars-in-query-res q-m st) + ;(def mc (mapcat (partial seqify-result-path ind) q-res )) + ;(map (comp(partial apply (:db-to-eav ind))(partial partition 2)) mc) +;(def aa1 (map #(->> %1 (partition 2)(apply (:db-to-eav ind))) mc) ) +;(reduce #(assoc-in %1 (butlast %2) (last %2)) {} aa1) + ;(merge-query-and-meta q-res ind) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) From 2f118c953d7948a310301b00cabbaaabf92cfb61 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Tue, 8 Apr 2014 18:34:10 +0300 Subject: [PATCH 078/102] refactoring for better naming, updating comments and general cleanup --- functionalDB/src/core/fdb.clj | 4 +-- functionalDB/src/core/manage.clj | 10 +++---- functionalDB/src/core/query.clj | 37 ++++++++++++------------- functionalDB/src/scenarios/hospital.clj | 16 ++--------- 4 files changed, 26 insertions(+), 41 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 17e288f1f..5b23a97e5 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -234,9 +234,9 @@ `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query ind# (choose-index ~db query#) ; selecting which index to use q-res# (query-index ind# query#) ; actually quering, from now on just preparing results - merged-res# (merge-query-and-meta q-res# ind#) ; building a meta + real results structure + binded-res# (bind-variables-to-query q-res# ind#) ; building a meta + real results structure needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables - (map (partial locate-vars-in-query-res needed-vars# ) merged-res#))) ; picking the needed variables from the query result + (map (partial locate-vars-in-query-res needed-vars# ) binded-res#))) ; picking the needed variables from the query result (defn evolution-of "The sequence of the values of of an entity's attribute, as changed through time" diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index 0aacd7f54..76a2b3ded 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,17 +1,17 @@ (ns core.manage - "Management of db and db connections via this module it is possible to either acquire new db or reset an existing one, as well as get the db value from a connection" + "Management of db creation and db connections. This module allows acquire new db or reset an existing one, as well as get the db value from a connection" (:use core.constructs)) (def __ALL-DBS__ (atom {})) -(defn _get-db [dbs db-name] - (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) +(defn _get-db [dbs db-name] (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) -(defn _reset-db[dbs db-name] (dissoc dbs db-name)) +(defn _reset-db [dbs db-name] (dissoc dbs db-name)) (defn _as-db-name [db-name] (keyword db-name)) -(defn get-db-conn [db-name] +(defn get-db-conn + [db-name] (let [stored-db-name (_as-db-name db-name)] (stored-db-name (swap! __ALL-DBS__ _get-db stored-db-name)))) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index 845ce3d37..2f9c44ddf 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -93,41 +93,38 @@ (set))) ; return it as set (defn mask-path-leaf-with-items - "a path is a vector triplet, at the first position there's a set of the path's items, here we intersect the path's items with the set of relevant items" - [index relevant-items path] - (update-in path [2] CS/intersection relevant-items ));) + "Returning the path with only the items found in the intersection of that path's items and the relevant items" + [relevant-items path] + (update-in path [2] CS/intersection relevant-items )) - (defn seqify-result-path + (defn seqify-index-path "A result-path is a path whose leaves are the items representing the items of the query chaining variable. This function returns for a result path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item is followed - by it's variable name as was inserted in the query" + by its variable name as was inserted in the query" [index path] (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] meta-path(apply (from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index all-path (interleave meta-path seq-path)] (apply (partial map vector) all-path))) -(defn merge-query-and-meta - "A function that receives the query results and restructues them to be a map, where - the key is a binding pair of a found entity-id, and the value is also a map, where its key is the binding pair of a found +(defn bind-variables-to-query + "A function that receives the query results and restructues them to be an binding structure which resembles in its shape to an entity. + The binding structure is a map whose key is a binding pair of a found entity-id, and the value is also a map, where its key is the binding pair of a found attribute, and the value is the binding pair of that found attribute's value" [q-res index] - (let [seq-res-path (mapcat (partial seqify-result-path index) q-res) ; seq-ing a result to hold the meta + (let [seq-res-path (mapcat (partial seqify-index-path index) q-res) ; seq-ing a result to hold the meta res-path (map #(->> %1 (partition 2)(apply (to-eav index))) seq-res-path)] ; making binding pairs - (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) ; structuring the pairs into the wanted map structure + (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) ; structuring the pairs into the wanted binding structure (defn query-index - "an index is a 3 level map (AVE, EAV, VEA) that is found at a specific time (T). - The path-preds argument is a seq of vectors. Each of these vectors contains two elements: - -- The first element is used as a filter predicator on the first level of the index (V -> any relevant function may be used, A -> an attribute name matching predicator, E -> an id matching predicator) - -- The second element is used as a filter on the second level of the index, and it is to be used for the branches of the index that passed the first element predicator. - -- to-eav is a function that receives a path in the index (3 elements ordered from root to leave in the index) and returns a vector containing these 3 elements, ordered like this [entity attr val] - The values found at the third level of the map of all the branches that passed the above two filters are fused into one seq" + "Quering an index based a seq of path predicates. A path predicate is composed of 3 predicates, each one to operate on a different level of the index. Querying an index with + a specific path-pred returns a result-path. We then take all the result paths and find within them the items that passed all the path-preds, and eventually return the result path, each contains + only the items that passed all the path predicates." [index path-preds] - (let [filtered-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path - relevant-items (items-that-answer-all-conditions (map last filtered-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths - relevant-paths (map #(mask-path-leaf-with-items index relevant-items %) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates - (filter #(not-empty (last %)) relevant-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items + (let [result-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path + relevant-items (items-that-answer-all-conditions (map last result-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths + cleaned-paths (map (partial mask-path-leaf-with-items relevant-items) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates + (filter #(not-empty (last %)) cleaned-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items (defn resultify-bind-pair "A bind pair is composed of two elements - the variable name and its value. Resultifying means to check whether the variable is suppose to be part of the diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index c0083cdbf..cab4b1fc4 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,5 +1,5 @@ (ns scenarios.hospital - (:use [core fdb query constructs] ) + (:use [core fdb constructs ] ) [:require [core.manage :as M] [clojure.set :as CS :only (union difference )]]) @@ -50,25 +50,13 @@ (defn keep-on-equals [a b](if (= a b) a nil )) -;(def qq(map #(:db/variable (meta %)) (q-clauses [[ ?e :test/bp-systolic (> 150 ?b)][ ?e :test/bp-diastolic ?k]] ))) -(ind-at @hospital-db :EAVT) (q @hospital-db {:find [?e ?k] :where [[ ?e :test/bp-systolic (> 200 ?b)] [ ?e :test/bp-diastolic ?k]]} ) -;(def ind (ind-at @hospital-db :VEAT)) -;(def q-m + (q @hospital-db {:find [?a ?b] :where [[ _ ?a (> 200 ?b)] ]}) (q @hospital-db {:find [?e ?k ] :where [[ ?e :test/bp-systolic (> 180 ?b)][ ?e :test/bp-diastolic ?k] ]}) - ;) - ;(map (partial vars-in-query-res st ) q-m) - -;(vars-in-query-res q-m st) - ;(def mc (mapcat (partial seqify-result-path ind) q-res )) - ;(map (comp(partial apply (:db-to-eav ind))(partial partition 2)) mc) -;(def aa1 (map #(->> %1 (partition 2)(apply (:db-to-eav ind))) mc) ) -;(reduce #(assoc-in %1 (butlast %2) (last %2)) {} aa1) - ;(merge-query-and-meta q-res ind) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) From 324c72fabaa507661f007751b8ac305c4cfbc134 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 9 Apr 2014 14:42:56 +0300 Subject: [PATCH 079/102] better docs --- functionalDB/src/core/query.clj | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index 2f9c44ddf..fd967c556 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -5,8 +5,8 @@ (defn variable? "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" ([x] (variable? x true)) - ([x accept_] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF - (or (and accept_ (= x "_")) (= (first x) \?)))) + ([x accept_?] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF + (or (and accept_? (= x "_")) (= (first x) \?)))) (defn ind-at "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EAVT" @@ -66,13 +66,11 @@ (ind-at db ind-to-use))) (defn filter-index - "Helper function to return only the sub-index (subsetting paths and leaves) that passes the given predicates. The result is a seq of triplet built as follows: - -- At index 0 : the key used at the first level of the path. - -- At index 1 : the key used at the second level of the path - -- At index 2 : the elements (e-ids or attribute names) that are found at the leave of the path." + "Function that accepts an index and a path-predicate (which is a tripet of predicates to apply on paths in an index). For each path predicates it creates a result path (a triplet + representing one path in the index) and returns a seq of result paths." [index path-preds] (for [ path-pred path-preds - :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta + :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta [k1 l2map] index ; keys and values of the first level :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate [k2 l3-set] l2map ; keys and values of the second level From 8aaa8c6c0337987eae797d70bc8b4d7d5156b5cd Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Wed, 9 Apr 2014 17:25:17 +0300 Subject: [PATCH 080/102] moving ind-at to be part of the main namespace --- functionalDB/src/core/fdb.clj | 7 +++++++ functionalDB/src/core/query.clj | 7 ------- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5b23a97e5..87faa9945 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -251,3 +251,10 @@ [db ts] (let [timestamped-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped timestamped-before :curr-time ts))) + +(defn ind-at + "inspecting a specific index at a given time, defaults to current. The kind argument mayone of the index name (e.g., AVET)" + ([db kind] + (ind-at db kind (:curr-time db))) + ([db kind ts] + (kind ((:timestamped db) ts)))) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index fd967c556..b438e0a9f 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -8,13 +8,6 @@ ([x accept_?] ; intentionally accepts a string and implemented as function and not a macro so we'd be able to use it as a HOF (or (and accept_? (= x "_")) (= (first x) \?)))) -(defn ind-at - "inspecting a specific index at a given time, defaults to current. The kind argument may be of of these: :AVET :VAET :EAVT" - ([db kind] - (ind-at db kind (:curr-time db))) - ([db kind ts] - (kind ((:timestamped db) ts)))) - (defmacro clause-item-meta "Finds the name of the variable at an item of a datalog clause element. If no variable, returning nil" [clause-item] From afddc8beff0f45dfa8a84576d4c5f7acce07617f Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 13 Apr 2014 17:18:17 +0300 Subject: [PATCH 081/102] clean-up after refactoring --- functionalDB/src/core/fdb.clj | 7 ------- functionalDB/src/core/query.clj | 9 ++++++++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 87faa9945..5b23a97e5 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -251,10 +251,3 @@ [db ts] (let [timestamped-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped timestamped-before :curr-time ts))) - -(defn ind-at - "inspecting a specific index at a given time, defaults to current. The kind argument mayone of the index name (e.g., AVET)" - ([db kind] - (ind-at db kind (:curr-time db))) - ([db kind ts] - (kind ((:timestamped db) ts)))) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index b438e0a9f..911470f84 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -2,6 +2,13 @@ [:use [core constructs] [clojure.set :as CS :only (intersection)]]) +(defn ind-at + "inspecting a specific index at a given time, defaults to current. The kind argument mayone of the index name (e.g., AVET)" + ([db kind] + (ind-at db kind (:curr-time db))) + ([db kind ts] + (kind ((:timestamped db) ts)))) + (defn variable? "A predicate that accepts a string and checks whether it describes a datalog variable (either starts with ? or it is _)" ([x] (variable? x true)) @@ -114,7 +121,7 @@ [index path-preds] (let [result-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path relevant-items (items-that-answer-all-conditions (map last result-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths - cleaned-paths (map (partial mask-path-leaf-with-items relevant-items) filtered-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates + cleaned-paths (map (partial mask-path-leaf-with-items relevant-items) result-paths)] ; the paths, now their leaves are filtered to have only the items that fulfilled the predicates (filter #(not-empty (last %)) cleaned-paths))) ; of these, we'll build a subset-path of the index that contains the paths to the leaves (sets), and these leaves contain only the valid items (defn resultify-bind-pair From ba3969437a759d0b8fb05c6e7efb3edb8b3d097d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 13 Apr 2014 18:52:56 +0300 Subject: [PATCH 082/102] fixing copy/paste error --- functionalDB/src/core/constructs.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/core/constructs.clj b/functionalDB/src/core/constructs.clj index d3d76dafa..e4b3befa1 100644 --- a/functionalDB/src/core/constructs.clj +++ b/functionalDB/src/core/constructs.clj @@ -17,7 +17,7 @@ (defn from-eav [index] (:from-eav (meta index))) (defn to-eav [index] (:to-eav (meta index))) (defn usage-pred [index] (:usage-pred (meta index))) -(defn leaf-index [index] (:usage-pred (meta leaf-index))) +(defn leaf-index [index] (:leaf-index (meta leaf-index))) (defn single? [attr] (= :db/single (:cardinality (meta attr)))) From 829d42191bf59a83c098e740a4e68ddc271243e2 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 14 Apr 2014 12:24:44 +0300 Subject: [PATCH 083/102] removing unneeded lines --- functionalDB/src/core/fdb.clj | 55 ++++++++----------------- functionalDB/src/core/manage.clj | 3 +- functionalDB/src/scenarios/hospital.clj | 4 +- 3 files changed, 20 insertions(+), 42 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5b23a97e5..726b0d1bc 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -55,8 +55,7 @@ [ent ts-val] (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) -(defn add-entry-to-index - [index path operation] +(defn add-entry-to-index [index path operation] (if (= operation :db/remove) index (let [update-path (butlast path) @@ -64,14 +63,12 @@ to-be-updated-set (get-in index update-path #{})] (assoc-in index update-path (conj to-be-updated-set update-value) )))) -(defn update-attr-in-index - [index ent-id attr-name target-val operation] +(defn update-attr-in-index [index ent-id attr-name target-val operation] (let [ colled-target-val (collify target-val) add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] (reduce add-entry-fn index colled-target-val))) -(defn add-entity-to-index - [ent timestamped ind-name] +(defn add-entity-to-index [ent timestamped ind-name] (let [ent-id (:id ent) index (ind-name timestamped) all-attrs (vals (:attrs ent)) @@ -79,14 +76,12 @@ add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) -(defn fix-new-entity - [db ent] +(defn fix-new-entity [db ent] (let [[ent-id next-top-id] (next-id db ent) new-ts (next-ts db)] [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) -(defn add-entity ;when adding an entity, its attributes' timestamp would be set to be the current one - [db ent] +(defn add-entity [db ent] (let [[fixed-ent next-top-id](fix-new-entity db ent) new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) add-fn (partial add-entity-to-index fixed-ent) @@ -95,12 +90,9 @@ (defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) -(defn update-attr-modification-time - [attr new-ts] - (assoc attr :ts new-ts :prev-ts ( :ts attr))) +(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) -(defn update-attr - [attr new-val new-ts operation] +(defn update-attr [attr new-val new-ts operation] {:pre [(if (single? attr) (contains? #{:db/reset-to :db/remove} operation) (contains? #{:db/reset-to :db/add :db/remove} operation))]} @@ -108,15 +100,13 @@ (update-attr-modification-time new-ts) (update-attr-value new-val operation))) -(defn remove-path-values - [old-vals target-val operation] +(defn remove-path-values [old-vals target-val operation] (cond (= operation :db/add) [] ; nothing to remove (= operation :db/reset-to) old-vals ; removing all of the old values (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller -(defn remove-entry-from-index - [index path] +(defn remove-entry-from-index [index path] (let [path-head (first path) path-to-items (butlast path) val-to-remove (last path) @@ -127,8 +117,7 @@ (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it :else (update-in index path-to-items disj val-to-remove)))) -(defn remove-entries-from-index - [ent-id operation index attr] +(defn remove-entries-from-index [ent-id operation index attr] (if (= operation :db/add) index (let [attr-name (:name attr) @@ -136,8 +125,7 @@ paths (map #((from-eav index) ent-id attr-name %) datom-vals)] (reduce remove-entry-from-index index paths)))) -(defn update-index - [ent-id old-attr target-val operation timestamped ind-name] +(defn update-index [ent-id old-attr target-val operation timestamped ind-name] (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) timestamped (let [index (ind-name timestamped) @@ -148,8 +136,7 @@ updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] (assoc timestamped ind-name updated-index)))) -(defn update-entity - [storage e-id new-attr] +(defn update-entity [storage e-id new-attr] (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) (defn update-timestamped @@ -169,8 +156,7 @@ fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] (update-in db [:timestamped] conj fully-updated-timestamped)))) -(defn remove-entity-from-index - [ent timestamped ind-name] +(defn remove-entity-from-index [ent timestamped ind-name] (let [ent-id (:id ent) index (ind-name timestamped) all-attrs (vals (:attrs ent)) @@ -178,22 +164,19 @@ remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) -(defn reffing-datoms-to - [e-id timestamped] +(defn reffing-datoms-to [e-id timestamped] (let [vaet (:VAET timestamped)] (for [[attr-name reffing-set] (e-id vaet) reffing reffing-set] [reffing attr-name e-id]))) -(defn remove-back-refs - [db e-id timestamped] +(defn remove-back-refs [db e-id timestamped] (let [refing-datoms (reffing-datoms-to e-id timestamped) remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) clean-db (reduce remove-fn db refing-datoms)] (last (:timestamped db)))) -(defn remove-entity - [db ent-id] +(defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) timestamped (remove-back-refs db ent-id (last (:timestamped db))) retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) @@ -201,8 +184,7 @@ new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) -(defn transact-on-db - [initial-db txs] +(defn transact-on-db [initial-db txs] (loop [[tx & rst-tx] txs transacted initial-db] (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) @@ -210,8 +192,7 @@ new-timestamped (last (:timestamped transacted))] (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) -(defmacro _transact - [db op & txs] +(defmacro _transact [db op & txs] (when txs (loop [[frst-tx# & rst-tx#] txs res# [op db `transact-on-db] accum-txs# []] (if frst-tx# diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index 76a2b3ded..6d1ae62cd 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -10,8 +10,7 @@ (defn _as-db-name [db-name] (keyword db-name)) -(defn get-db-conn - [db-name] +(defn get-db-conn [db-name] (let [stored-db-name (_as-db-name db-name)] (stored-db-name (swap! __ALL-DBS__ _get-db stored-db-name)))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index cab4b1fc4..c949e89e1 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -30,15 +30,13 @@ (defn add-test-results-to-patient [pat-id test-result] (let [test-id (:id test-result) - a (transact hospital-db (add-entity test-result)) - ] + a (transact hospital-db (add-entity test-result))] (transact hospital-db (update-datom pat-id :patient/tests #{test-id} :db/add)))) ;; world setup (transact hospital-db (add-entities (map #(make-entity %) basic-kinds ))) (add-patient :pat1 "London" ["fever" "cough"] ) - (add-patient :pat2 "London" ["fever" "cough"] ) (add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} )) From 74f2c9eb9f6d8a67d7ff1d6b1f2fb101087be7c2 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Thu, 17 Apr 2014 11:39:46 +0300 Subject: [PATCH 084/102] improving readability by changing cond to case --- functionalDB/src/core/fdb.clj | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 726b0d1bc..d6f6c1cbb 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -101,10 +101,10 @@ (update-attr-value new-val operation))) (defn remove-path-values [old-vals target-val operation] - (cond - (= operation :db/add) [] ; nothing to remove - (= operation :db/reset-to) old-vals ; removing all of the old values - (= operation :db/remove) (collify target-val))) ; removing the values defined by the caller + (case operation + :db/add [] ; nothing to remove + :db/reset-to old-vals ; removing all of the old values + :db/remove (collify target-val))) ; removing the values defined by the caller (defn remove-entry-from-index [index path] (let [path-head (first path) From 813d4b9a8c6dd411e8eedeaef9efd21ec5758ab4 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 19 Apr 2014 17:57:10 +0300 Subject: [PATCH 085/102] adding db-traversals - treating the database as graph --- functionalDB/src/core/fdb.clj | 32 ++++++++++++++++++++++--- functionalDB/src/scenarios/hospital.clj | 6 +++++ 2 files changed, 35 insertions(+), 3 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index d6f6c1cbb..6975bb954 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -19,15 +19,15 @@ (defn entity-at "the entity with the given ent-id at the given time (defualts to the latest time)" - ([db ent-id] (entity-at db ent-id (:curr-time db))) - ([db ent-id ts] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) + ([db ent-id] (entity-at db (:curr-time db) ent-id)) + ([db ts ent-id] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) (defn attr-at "The attribute of an entity at a given time (defaults to recent time)" ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] - (get-in (entity-at db ent-id ts) [:attrs attr-name]))) + (get-in (entity-at db ts ent-id) [:attrs attr-name]))) (defn value-of-at "value of a datom at a given time, if no time is provided, we default to the most recent value" @@ -232,3 +232,29 @@ [db ts] (let [timestamped-before (subvec (:timestamped db) 0 ts )] (assoc db :timestamped timestamped-before :curr-time ts))) + +(defn outgoing-refs [db ts ent-id] + (if ent-id + (->> (entity-at db ent-id ts) + (:attrs) + (vals) + (filter ref?) + (mapcat (comp collify :value))) + [])) + +(defn remove-explored [pendings explored] + (if (contains? explored (first pendings)) + (recur (rest pendings) explored) + pendings)) + +(defn traverse [pendings explored out-reffing ent-at] + (let [cleaned-pendings (remove-explored pendings explored) + item (first cleaned-pendings) + next-pends (reduce conj (rest cleaned-pendings) (out-reffing item))] + (when item (cons (ent-at item) + (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at)))))) + +(defn traverse-db + ([start-ent-id db bfs?] (traverse-db start-ent-id db (:curr-time db) bfs-or-dfs)) + ([start-ent-id db ts bfs-or-dfs] + (traverse (if (= :bfs bfs-or-dfs) [start-ent-id] '(start-ent-id)) #{} (partial outgoing-refs db ts) (partial entity-at db ts)))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index c949e89e1..cbc803cb5 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -59,3 +59,9 @@ (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/tests) + + + (take 4 (traverse-db :pat2 @hospital-db (:curr-time @hospital-db) :bfs)) + + (outgoing-refs @hospital-db 9 :pat2) +;(entity-at @hospital-db :pat1 9) From 927e423a3f4ed905a1e06151b45075adcd2d5737 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 19 Apr 2014 18:06:23 +0300 Subject: [PATCH 086/102] fixing refactoring typo --- functionalDB/src/core/fdb.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 6975bb954..5ef9efdc2 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -235,7 +235,7 @@ (defn outgoing-refs [db ts ent-id] (if ent-id - (->> (entity-at db ent-id ts) + (->> (entity-at db ts ent-id) (:attrs) (vals) (filter ref?) @@ -255,6 +255,6 @@ (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at)))))) (defn traverse-db - ([start-ent-id db bfs?] (traverse-db start-ent-id db (:curr-time db) bfs-or-dfs)) + ([start-ent-id db bfs-or-dfs] (traverse-db start-ent-id db (:curr-time db) bfs-or-dfs)) ([start-ent-id db ts bfs-or-dfs] (traverse (if (= :bfs bfs-or-dfs) [start-ent-id] '(start-ent-id)) #{} (partial outgoing-refs db ts) (partial entity-at db ts)))) From bcee9a9a39b55ae97b6c4e51532f8a7075209019 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sun, 20 Apr 2014 14:04:19 +0300 Subject: [PATCH 087/102] adding the traversal either of outgoing or incoming refs bug fix in the algorithm implementation imporoved scenario --- functionalDB/src/core/fdb.clj | 40 +++++++++++++++---------- functionalDB/src/scenarios/hospital.clj | 18 ++++++++--- 2 files changed, 39 insertions(+), 19 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5ef9efdc2..5a51bd252 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -236,25 +236,35 @@ (defn outgoing-refs [db ts ent-id] (if ent-id (->> (entity-at db ts ent-id) - (:attrs) - (vals) - (filter ref?) - (mapcat (comp collify :value))) + (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) [])) -(defn remove-explored [pendings explored] - (if (contains? explored (first pendings)) - (recur (rest pendings) explored) - pendings)) +(defn incoming-refs [db ts ent-id] + (let [vaet (ind-at db ts :VAET)] + (reduce into (vals (ent-id vaet))))) -(defn traverse [pendings explored out-reffing ent-at] - (let [cleaned-pendings (remove-explored pendings explored) +(defn remove-explored [pendings explored restruct-fn] + (if (contains? explored (first pendings)) + (recur (rest pendings) explored restruct-fn) + (restruct-fn pendings))) + +(defn traverse [pendings explored out-reffing ent-at restruct-fn] + (let [_ (println "\n\nexplo" explored) + _ (println "all pends " pendings) + cleaned-pendings (remove-explored pendings explored restruct-fn) + _ (println "cleaned = " cleaned-pendings) item (first cleaned-pendings) - next-pends (reduce conj (rest cleaned-pendings) (out-reffing item))] + _ (println "item ==" item) + next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item)) + + ;stam (println "cleaned pends" next-pends) + ] (when item (cons (ent-at item) - (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at)))))) + (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at restruct-fn)))))) (defn traverse-db - ([start-ent-id db bfs-or-dfs] (traverse-db start-ent-id db (:curr-time db) bfs-or-dfs)) - ([start-ent-id db ts bfs-or-dfs] - (traverse (if (= :bfs bfs-or-dfs) [start-ent-id] '(start-ent-id)) #{} (partial outgoing-refs db ts) (partial entity-at db ts)))) + ([start-ent-id db algo direction] (traverse-db start-ent-id db algo direction (:curr-time db))) + ([start-ent-id db algo direction ts] + (let [pend-struct (if (= :bfs algo) vec list*) + explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] + (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index cbc803cb5..ebf81e282 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -25,6 +25,11 @@ :indexed true ));indexed ent tests-map))) +(defn add-machine [id nm] + (transact hospital-db (add-entity (-> (make-entity id) + (add-attr (make-attr :machine/name nm :string)))))) + + (defn add-patient [id address symptoms] (transact hospital-db (add-entity (make-patient id address symptoms)))) @@ -39,8 +44,12 @@ (add-patient :pat1 "London" ["fever" "cough"] ) (add-patient :pat2 "London" ["fever" "cough"] ) -(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine "XXX"} {:test/machine "string"} )) -(add-test-results-to-patient :pat2 (make-test :t4-pat1 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine "XYY"} {:test/machine "string"} )) +(add-machine :1machine1 "M11") +(add-machine :2machine2 "M222") + +(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine :2machine2 } {:test/machine :db/ref} )) +(add-test-results-to-patient :pat2 (make-test :t4-pat2 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine :1machine1} {:test/machine :db/ref} )) +(add-test-results-to-patient :pat2 (make-test :t3-pat2 {:test/bp-systolic 140 :test/bp-diastolic 80 :test/machine :2machine2} {:test/machine :db/ref} )) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) (transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) @@ -61,7 +70,8 @@ - (take 4 (traverse-db :pat2 @hospital-db (:curr-time @hospital-db) :bfs)) + (take 7 + (traverse-db :pat2 @hospital-db :bfs :outgoing)) + - (outgoing-refs @hospital-db 9 :pat2) ;(entity-at @hospital-db :pat1 9) From ebb3817311b460664c1e122e900e41b6e6dfe661 Mon Sep 17 00:00:00 2001 From: yrubin Date: Tue, 29 Apr 2014 16:11:07 +0300 Subject: [PATCH 088/102] unneeded creation of local, it is enough to return the result of the if --- functionalDB/src/core/fdb.clj | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5a51bd252..6abde0a91 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -11,11 +11,10 @@ [db ent] (let [top-id (:top-id db) ent-id (:id ent) - inceased-id (inc top-id) - [id-to-use next-top] (if (= ent-id :db/no-id-yet) - [(keyword (str inceased-id)) inceased-id] - [ent-id top-id])] - [id-to-use next-top])) + inceased-id (inc top-id)] + (if (= ent-id :db/no-id-yet) + [(keyword (str inceased-id)) inceased-id] + [ent-id top-id]))) (defn entity-at "the entity with the given ent-id at the given time (defualts to the latest time)" From 8033e7ef864d4a154c2f28dbb37adbb23e0f2b6e Mon Sep 17 00:00:00 2001 From: yrubin Date: Mon, 5 May 2014 10:10:21 +0300 Subject: [PATCH 089/102] following @beloglazov comments - better and idiomatic usage of defn- with better naming, thus getting a clearer cut between the public and the private APIs of this namespace. --- functionalDB/src/core/manage.clj | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index 6d1ae62cd..ad32652a5 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -1,21 +1,21 @@ (ns core.manage - "Management of db creation and db connections. This module allows acquire new db or reset an existing one, as well as get the db value from a connection" + "Management of db connections (from the user's perspective), internally creates/ / drops dbs." (:use core.constructs)) -(def __ALL-DBS__ (atom {})) +(def ^:private __ALL-DBS__ (atom {})) -(defn _get-db [dbs db-name] (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) +(defn- force-get-db [dbs db-name] (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) -(defn _reset-db [dbs db-name] (dissoc dbs db-name)) +(defn- drop-db [dbs db-name] (dissoc dbs db-name)) -(defn _as-db-name [db-name] (keyword db-name)) +(defn- as-db-name [db-name] (keyword db-name)) (defn get-db-conn [db-name] - (let [stored-db-name (_as-db-name db-name)] - (stored-db-name (swap! __ALL-DBS__ _get-db stored-db-name)))) + (let [stored-db-name (as-db-name db-name)] + (stored-db-name (swap! __ALL-DBS__ force-get-db stored-db-name)))) -(defn reset-db-conn [db-name] - (let [stored-db-name (_as-db-name db-name)] - (swap! __ALL-DBS__ _reset-db stored-db-name)) nil) +(defn drop-db-conn [db-name] + (let [stored-db-name (as-db-name db-name)] + (swap! __ALL-DBS__ drop-db stored-db-name)) nil) (defn db-from-conn [conn] @conn) From 8f1db79041f94250741a1f327e43a2b0a184c8c6 Mon Sep 17 00:00:00 2001 From: yrubin Date: Mon, 5 May 2014 10:22:58 +0300 Subject: [PATCH 090/102] fixed typos and remove trace/debug code --- functionalDB/src/core/constructs.clj | 4 ++-- functionalDB/src/core/fdb.clj | 13 +++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/functionalDB/src/core/constructs.clj b/functionalDB/src/core/constructs.clj index e4b3befa1..02639e3e0 100644 --- a/functionalDB/src/core/constructs.clj +++ b/functionalDB/src/core/constructs.clj @@ -42,10 +42,10 @@ ([id] (Entity. id {}))) (defn make-attr - "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguguments. + "creation of an attribute. The name, value and type of an attribute are mandatory arguments, further arguments can be passed as named arguments. The type of the attribute may be either :string, :number, :boolean or :db/ref . If the type is :db/ref, the value is an id of another entity and indexing of backpointing is maintained. The named arguments are as follows: - :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not inexed. + :indexed - a boolean, can be either true or false - marks whether this attribute should be indexed. By defaults attributes are not indexed. :cardinality - the cardinality of an attribute, can be either: :db/single - which means that this attribute can be a single value at any given time (this is the default cardinality) :db/multiple - which means that this attribute is actually a set of values. In this case updates of this attribute may be one of the following (NOTE that all these operations accept a set as argument): diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 6abde0a91..5a1b6b58e 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -248,16 +248,9 @@ (restruct-fn pendings))) (defn traverse [pendings explored out-reffing ent-at restruct-fn] - (let [_ (println "\n\nexplo" explored) - _ (println "all pends " pendings) - cleaned-pendings (remove-explored pendings explored restruct-fn) - _ (println "cleaned = " cleaned-pendings) - item (first cleaned-pendings) - _ (println "item ==" item) - next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item)) - - ;stam (println "cleaned pends" next-pends) - ] + (let [cleaned-pendings (remove-explored pendings explored restruct-fn) + item (first cleaned-pendings) + next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item))] (when item (cons (ent-at item) (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at restruct-fn)))))) From c30b5e8dcd09ce5c680f388c064996156b98bb52 Mon Sep 17 00:00:00 2001 From: yrubin Date: Mon, 5 May 2014 10:31:06 +0300 Subject: [PATCH 091/102] formatting based on @beloglazov comments --- functionalDB/src/scenarios/hospital.clj | 27 ++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index ebf81e282..7c1048763 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -47,9 +47,16 @@ (add-machine :1machine1 "M11") (add-machine :2machine2 "M222") -(add-test-results-to-patient :pat1 (make-test :t2-pat1 {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine :2machine2 } {:test/machine :db/ref} )) -(add-test-results-to-patient :pat2 (make-test :t4-pat2 {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine :1machine1} {:test/machine :db/ref} )) -(add-test-results-to-patient :pat2 (make-test :t3-pat2 {:test/bp-systolic 140 :test/bp-diastolic 80 :test/machine :2machine2} {:test/machine :db/ref} )) +(add-test-results-to-patient :pat1 + (make-test :t2-pat1 + {:test/bp-systolic 170 :test/bp-diastolic 80 :test/machine :2machine2 } + {:test/machine :db/ref} )) +(add-test-results-to-patient :pat2 (make-test :t4-pat2 + {:test/bp-systolic 170 :test/bp-diastolic 90 :test/machine :1machine1} + {:test/machine :db/ref} )) +(add-test-results-to-patient :pat2 (make-test :t3-pat2 + {:test/bp-systolic 140 :test/bp-diastolic 80 :test/machine :2machine2} + {:test/machine :db/ref} )) (transact hospital-db (update-datom :pat1 :patient/symptoms #{"cold sweat" "sneeze"} :db/reset-to)) (transact hospital-db (update-datom :pat1 :patient/tests #{:t2-pat1} :db/remove)) @@ -57,12 +64,17 @@ (defn keep-on-equals [a b](if (= a b) a nil )) -(q @hospital-db {:find [?e ?k] :where [[ ?e :test/bp-systolic (> 200 ?b)] [ ?e :test/bp-diastolic ?k]]} ) +(q @hospital-db {:find [?e ?k] + :where [[ ?e :test/bp-systolic (> 200 ?b)] + [ ?e :test/bp-diastolic ?k]]} ) - (q @hospital-db {:find [?a ?b] :where [[ _ ?a (> 200 ?b)] ]}) + (q @hospital-db {:find [?a ?b] + :where [[ _ ?a (> 200 ?b)]]}) - (q @hospital-db {:find [?e ?k ] :where [[ ?e :test/bp-systolic (> 180 ?b)][ ?e :test/bp-diastolic ?k] ]}) + (q @hospital-db {:find [?e ?k ] + :where [[ ?e :test/bp-systolic (> 180 ?b)] + [ ?e :test/bp-diastolic ?k] ]}) (evolution-of (M/db-from-conn hospital-db) :pat1 :patient/symptoms) @@ -72,6 +84,3 @@ (take 7 (traverse-db :pat2 @hospital-db :bfs :outgoing)) - - -;(entity-at @hospital-db :pat1 9) From c66ae82885d59efe8444afb8710d06e9dd61460d Mon Sep 17 00:00:00 2001 From: yrubin Date: Wed, 7 May 2014 08:57:26 +0300 Subject: [PATCH 092/102] alignment --- functionalDB/src/core/query.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index 911470f84..3bf8d9dfc 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -4,7 +4,7 @@ (defn ind-at "inspecting a specific index at a given time, defaults to current. The kind argument mayone of the index name (e.g., AVET)" - ([db kind] + ([db kind] (ind-at db kind (:curr-time db))) ([db kind ts] (kind ((:timestamped db) ts)))) From 3f21d0543bec4a4d52584e11bdebb6a9df4ff084 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 9 May 2014 17:44:36 +0300 Subject: [PATCH 093/102] merging --- .gitignore | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index f5ceb8409..ca9767ae0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ -*~ -_site - -.project -functionalDB/node_modules - -*.pyc - -/bin +*~ +_site +*.pyc +.coverage +htmlcov \ No newline at end of file From d0314fd53d6943b22476c494f76bd17a049101ff Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Fri, 9 May 2014 21:48:34 +0300 Subject: [PATCH 094/102] update following renaming --- functionalDB/src/scenarios/hospital.clj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index 7c1048763..e07f6edb4 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -4,7 +4,7 @@ [clojure.set :as CS :only (union difference )]]) (def db-name "hos12") -(M/reset-db-conn db-name) +(M/drop-db-conn db-name) (def hospital-db (M/get-db-conn db-name)) From ad3c96dcb246b63e4923a173a07ec459d7d5588d Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 10 May 2014 11:13:34 +0300 Subject: [PATCH 095/102] unneeded space --- functionalDB/src/core/fdb.clj | 524 +++++++++++++++++----------------- 1 file changed, 262 insertions(+), 262 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 5a1b6b58e..941b79ce9 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,262 +1,262 @@ -(ns core.fdb - [:use [core storage query constructs]] - [:require [clojure.set :as CS :only (union difference intersection)]]) - -(defn collify [x] (if (coll? x) x [x])) - -(defn next-ts [db] (inc (:curr-time db))) - -(defn next-id - "returns a pair composed of the id to use for the given entity and the next free running id in the database" - [db ent] - (let [top-id (:top-id db) - ent-id (:id ent) - inceased-id (inc top-id)] - (if (= ent-id :db/no-id-yet) - [(keyword (str inceased-id)) inceased-id] - [ent-id top-id]))) - -(defn entity-at - "the entity with the given ent-id at the given time (defualts to the latest time)" - ([db ent-id] (entity-at db (:curr-time db) ent-id)) - ([db ts ent-id] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) - -(defn attr-at - "The attribute of an entity at a given time (defaults to recent time)" - ([db ent-id attr-name] - (attr-at db ent-id attr-name (:curr-time db))) - ([db ent-id attr-name ts] - (get-in (entity-at db ts ent-id) [:attrs attr-name]))) - -(defn value-of-at - "value of a datom at a given time, if no time is provided, we default to the most recent value" - ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) - ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) - -(defn update-attr-value - "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" - [attr value operation] - (cond - (single? attr) (assoc attr :value #{value}) - ; now we're talking about an attribute of multiple values - (= :db/reset-to operation) (assoc attr :value value) - (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) - (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) - -(defn add-attr - "adds an attribute to an entity" - [ent attr] - (let [attr-id (keyword (:name attr))] - (assoc-in ent [:attrs attr-id] attr))) - -(defn update-creation-ts - "updates the timestamp value of all the attributes of an entity to the given timestamp" - [ent ts-val] - (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) - -(defn add-entry-to-index [index path operation] - (if (= operation :db/remove) - index - (let [update-path (butlast path) - update-value (last path) - to-be-updated-set (get-in index update-path #{})] - (assoc-in index update-path (conj to-be-updated-set update-value) )))) - -(defn update-attr-in-index [index ent-id attr-name target-val operation] - (let [ colled-target-val (collify target-val) - add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] - (reduce add-entry-fn index colled-target-val))) - -(defn add-entity-to-index [ent timestamped ind-name] - (let [ent-id (:id ent) - index (ind-name timestamped) - all-attrs (vals (:attrs ent)) - relevant-attrs (filter #((usage-pred index) %) all-attrs ) - add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] - (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) - -(defn fix-new-entity [db ent] - (let [[ent-id next-top-id] (next-id db ent) - new-ts (next-ts db)] - [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) - -(defn add-entity [db ent] - (let [[fixed-ent next-top-id](fix-new-entity db ent) - new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) - add-fn (partial add-entity-to-index fixed-ent) - new-timestamped (reduce add-fn new-timestamped (indices))] - (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) - -(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) - -(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) - -(defn update-attr [attr new-val new-ts operation] - {:pre [(if (single? attr) - (contains? #{:db/reset-to :db/remove} operation) - (contains? #{:db/reset-to :db/add :db/remove} operation))]} - (-> attr - (update-attr-modification-time new-ts) - (update-attr-value new-val operation))) - -(defn remove-path-values [old-vals target-val operation] - (case operation - :db/add [] ; nothing to remove - :db/reset-to old-vals ; removing all of the old values - :db/remove (collify target-val))) ; removing the values defined by the caller - -(defn remove-entry-from-index [index path] - (let [path-head (first path) - path-to-items (butlast path) - val-to-remove (last path) - old-entries-set (get-in index path-to-items)] - (cond - (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here - (and (= 1 (count old-entries-set) ) (= 1 (count (index path-head)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely - (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it - :else (update-in index path-to-items disj val-to-remove)))) - -(defn remove-entries-from-index [ent-id operation index attr] - (if (= operation :db/add) - index - (let [attr-name (:name attr) - datom-vals (collify (:value attr)) - paths (map #((from-eav index) ent-id attr-name %) datom-vals)] - (reduce remove-entry-from-index index paths)))) - -(defn update-index [ent-id old-attr target-val operation timestamped ind-name] - (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) - timestamped - (let [index (ind-name timestamped) - old-vals (:value old-attr) - attr-name (:name old-attr) - remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index ent-id operation index old-attr) - updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] - (assoc timestamped ind-name updated-index)))) - -(defn update-entity [storage e-id new-attr] - (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) - -(defn update-timestamped - [timestamped ent-id old-attr updated-attr new-val operation] - (let [storage (:storage timestamped) - new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] - (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) - -(defn update-datom - ([db ent-id attr-name new-val] - (update-datom db ent-id attr-name new-val :db/reset-to )) - ([db ent-id attr-name new-val operation ] - (let [update-ts (next-ts db) - timestamped (last (:timestamped db)) - attr (attr-at db ent-id attr-name) - updated-attr (update-attr attr new-val update-ts operation) - fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] - (update-in db [:timestamped] conj fully-updated-timestamped)))) - -(defn remove-entity-from-index [ent timestamped ind-name] - (let [ent-id (:id ent) - index (ind-name timestamped) - all-attrs (vals (:attrs ent)) - relevant-attrs (filter #((usage-pred index) %) all-attrs ) - remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] - (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) - -(defn reffing-datoms-to [e-id timestamped] - (let [vaet (:VAET timestamped)] - (for [[attr-name reffing-set] (e-id vaet) - reffing reffing-set] - [reffing attr-name e-id]))) - -(defn remove-back-refs [db e-id timestamped] - (let [refing-datoms (reffing-datoms-to e-id timestamped) - remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) - clean-db (reduce remove-fn db refing-datoms)] - (last (:timestamped db)))) - -(defn remove-entity [db ent-id] - (let [ent (entity-at db ent-id) - timestamped (remove-back-refs db ent-id (last (:timestamped db))) - retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) - no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) - new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] - (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) - -(defn transact-on-db [initial-db txs] - (loop [[tx & rst-tx] txs transacted initial-db] - (if tx - (recur rst-tx (apply (first tx) transacted (rest tx))) - (let [initial-timestamped (:timestamped initial-db) - new-timestamped (last (:timestamped transacted))] - (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) - -(defmacro _transact [db op & txs] - (when txs - (loop [[frst-tx# & rst-tx#] txs res# [op db `transact-on-db] accum-txs# []] - (if frst-tx# - (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) - (list* (conj res# accum-txs#)))))) - -(defn _what-if - "Operates on the db with the given transactions, but without eventually updating it" - [ db f txs] - (f db txs)) - -(defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) - -(defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) - -(defmacro q - "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). - At the moment support only filtering queries, no joins is also assumed." - [db query] - `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query - ind# (choose-index ~db query#) ; selecting which index to use - q-res# (query-index ind# query#) ; actually quering, from now on just preparing results - binded-res# (bind-variables-to-query q-res# ind#) ; building a meta + real results structure - needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables - (map (partial locate-vars-in-query-res needed-vars# ) binded-res#))) ; picking the needed variables from the query result - -(defn evolution-of - "The sequence of the values of of an entity's attribute, as changed through time" - [db ent-id attr-name] - (loop [res [] ts (:curr-time db)] - (if (= -1 ts) (reverse res) - (let [attr (attr-at db ent-id attr-name ts)] - (recur (conj res {(:ts attr) (:value attr)}) (:prev-ts attr)))))) - -(defn db-before - "How the db was before a given timestamp" - [db ts] - (let [timestamped-before (subvec (:timestamped db) 0 ts )] - (assoc db :timestamped timestamped-before :curr-time ts))) - -(defn outgoing-refs [db ts ent-id] - (if ent-id - (->> (entity-at db ts ent-id) - (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) - [])) - -(defn incoming-refs [db ts ent-id] - (let [vaet (ind-at db ts :VAET)] - (reduce into (vals (ent-id vaet))))) - -(defn remove-explored [pendings explored restruct-fn] - (if (contains? explored (first pendings)) - (recur (rest pendings) explored restruct-fn) - (restruct-fn pendings))) - -(defn traverse [pendings explored out-reffing ent-at restruct-fn] - (let [cleaned-pendings (remove-explored pendings explored restruct-fn) - item (first cleaned-pendings) - next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item))] - (when item (cons (ent-at item) - (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at restruct-fn)))))) - -(defn traverse-db - ([start-ent-id db algo direction] (traverse-db start-ent-id db algo direction (:curr-time db))) - ([start-ent-id db algo direction ts] - (let [pend-struct (if (= :bfs algo) vec list*) - explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] - (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) +(ns core.fdb + [:use [core storage query constructs]] + [:require [clojure.set :as CS :only (union difference intersection)]]) + +(defn collify [x] (if (coll? x) x [x])) + +(defn next-ts [db] (inc (:curr-time db))) + +(defn next-id + "returns a pair composed of the id to use for the given entity and the next free running id in the database" + [db ent] + (let [top-id (:top-id db) + ent-id (:id ent) + inceased-id (inc top-id)] + (if (= ent-id :db/no-id-yet) + [(keyword (str inceased-id)) inceased-id] + [ent-id top-id]))) + +(defn entity-at + "the entity with the given ent-id at the given time (defualts to the latest time)" + ([db ent-id] (entity-at db (:curr-time db) ent-id)) + ([db ts ent-id] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) + +(defn attr-at + "The attribute of an entity at a given time (defaults to recent time)" + ([db ent-id attr-name] + (attr-at db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name ts] + (get-in (entity-at db ts ent-id) [:attrs attr-name]))) + +(defn value-of-at + "value of a datom at a given time, if no time is provided, we default to the most recent value" + ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) + ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) + +(defn update-attr-value + "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" + [attr value operation] + (cond + (single? attr) (assoc attr :value #{value}) + ; now we're talking about an attribute of multiple values + (= :db/reset-to operation) (assoc attr :value value) + (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) + (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) + +(defn add-attr + "adds an attribute to an entity" + [ent attr] + (let [attr-id (keyword (:name attr))] + (assoc-in ent [:attrs attr-id] attr))) + +(defn update-creation-ts + "updates the timestamp value of all the attributes of an entity to the given timestamp" + [ent ts-val] + (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) + +(defn add-entry-to-index [index path operation] + (if (= operation :db/remove) + index + (let [update-path (butlast path) + update-value (last path) + to-be-updated-set (get-in index update-path #{})] + (assoc-in index update-path (conj to-be-updated-set update-value) )))) + +(defn update-attr-in-index [index ent-id attr-name target-val operation] + (let [ colled-target-val (collify target-val) + add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] + (reduce add-entry-fn index colled-target-val))) + +(defn add-entity-to-index [ent timestamped ind-name] + (let [ent-id (:id ent) + index (ind-name timestamped) + all-attrs (vals (:attrs ent)) + relevant-attrs (filter #((usage-pred index) %) all-attrs ) + add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] + (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) + +(defn fix-new-entity [db ent] + (let [[ent-id next-top-id] (next-id db ent) + new-ts (next-ts db)] + [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) + +(defn add-entity [db ent] + (let [[fixed-ent next-top-id](fix-new-entity db ent) + new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) + add-fn (partial add-entity-to-index fixed-ent) + new-timestamped (reduce add-fn new-timestamped (indices))] + (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) + +(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) + +(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) + +(defn update-attr [attr new-val new-ts operation] + {:pre [(if (single? attr) + (contains? #{:db/reset-to :db/remove} operation) + (contains? #{:db/reset-to :db/add :db/remove} operation))]} + (-> attr + (update-attr-modification-time new-ts) + (update-attr-value new-val operation))) + +(defn remove-path-values [old-vals target-val operation] + (case operation + :db/add [] ; nothing to remove + :db/reset-to old-vals ; removing all of the old values + :db/remove (collify target-val))) ; removing the values defined by the caller + +(defn remove-entry-from-index [index path] + (let [path-head (first path) + path-to-items (butlast path) + val-to-remove (last path) + old-entries-set (get-in index path-to-items)] + (cond + (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here + (and (= 1 (count old-entries-set) ) (= 1 (count (index path-head)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely + (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it + :else (update-in index path-to-items disj val-to-remove)))) + +(defn remove-entries-from-index [ent-id operation index attr] + (if (= operation :db/add) + index + (let [attr-name (:name attr) + datom-vals (collify (:value attr)) + paths (map #((from-eav index) ent-id attr-name %) datom-vals)] + (reduce remove-entry-from-index index paths)))) + +(defn update-index [ent-id old-attr target-val operation timestamped ind-name] + (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) + timestamped + (let [index (ind-name timestamped) + old-vals (:value old-attr) + attr-name (:name old-attr) + remove-paths (remove-path-values old-vals target-val operation) + cleaned-index (remove-entries-from-index ent-id operation index old-attr) + updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] + (assoc timestamped ind-name updated-index)))) + +(defn update-entity [storage e-id new-attr] + (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) + +(defn update-timestamped + [timestamped ent-id old-attr updated-attr new-val operation] + (let [storage (:storage timestamped) + new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] + (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) + +(defn update-datom + ([db ent-id attr-name new-val] + (update-datom db ent-id attr-name new-val :db/reset-to )) + ([db ent-id attr-name new-val operation ] + (let [update-ts (next-ts db) + timestamped (last (:timestamped db)) + attr (attr-at db ent-id attr-name) + updated-attr (update-attr attr new-val update-ts operation) + fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] + (update-in db [:timestamped] conj fully-updated-timestamped)))) + +(defn remove-entity-from-index [ent timestamped ind-name] + (let [ent-id (:id ent) + index (ind-name timestamped) + all-attrs (vals (:attrs ent)) + relevant-attrs (filter #((usage-pred index) %) all-attrs ) + remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] + (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) + +(defn reffing-datoms-to [e-id timestamped] + (let [vaet (:VAET timestamped)] + (for [[attr-name reffing-set] (e-id vaet) + reffing reffing-set] + [reffing attr-name e-id]))) + +(defn remove-back-refs [db e-id timestamped] + (let [refing-datoms (reffing-datoms-to e-id timestamped) + remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) + clean-db (reduce remove-fn db refing-datoms)] + (last (:timestamped db)))) + +(defn remove-entity [db ent-id] + (let [ent (entity-at db ent-id) + timestamped (remove-back-refs db ent-id (last (:timestamped db))) + retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) + no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) + new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] + (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) + +(defn transact-on-db [initial-db txs] + (loop [[tx & rst-tx] txs transacted initial-db] + (if tx + (recur rst-tx (apply (first tx) transacted (rest tx))) + (let [initial-timestamped (:timestamped initial-db) + new-timestamped (last (:timestamped transacted))] + (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) + +(defmacro _transact [db op & txs] + (when txs + (loop [[frst-tx# & rst-tx#] txs res# [op db `transact-on-db] accum-txs# []] + (if frst-tx# + (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) + (list* (conj res# accum-txs#)))))) + +(defn _what-if + "Operates on the db with the given transactions, but without eventually updating it" + [ db f txs] + (f db txs)) + +(defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) + +(defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) + +(defmacro q + "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). + At the moment support only filtering queries, no joins is also assumed." + [db query] + `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query + ind# (choose-index ~db query#) ; selecting which index to use + q-res# (query-index ind# query#) ; actually quering, from now on just preparing results + binded-res# (bind-variables-to-query q-res# ind#) ; building a meta + real results structure + needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables + (map (partial locate-vars-in-query-res needed-vars# ) binded-res#))) ; picking the needed variables from the query result + +(defn evolution-of + "The sequence of the values of of an entity's attribute, as changed through time" + [db ent-id attr-name] + (loop [res [] ts (:curr-time db)] + (if (= -1 ts) (reverse res) + (let [attr (attr-at db ent-id attr-name ts)] + (recur (conj res {(:ts attr) (:value attr)}) (:prev-ts attr)))))) + +(defn db-before + "How the db was before a given timestamp" + [db ts] + (let [timestamped-before (subvec (:timestamped db) 0 ts )] + (assoc db :timestamped timestamped-before :curr-time ts))) + +(defn outgoing-refs [db ts ent-id] + (if ent-id + (->> (entity-at db ts ent-id) + (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) + [])) + +(defn incoming-refs [db ts ent-id] + (let [vaet (ind-at db ts :VAET)] + (reduce into (vals (ent-id vaet))))) + +(defn remove-explored [pendings explored restruct-fn] + (if (contains? explored (first pendings)) + (recur (rest pendings) explored restruct-fn) + (restruct-fn pendings))) + +(defn traverse [pendings explored out-reffing ent-at restruct-fn] + (let [cleaned-pendings (remove-explored pendings explored restruct-fn) + item (first cleaned-pendings) + next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item))] + (when item (cons (ent-at item) + (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at restruct-fn)))))) + +(defn traverse-db + ([start-ent-id db algo direction] (traverse-db start-ent-id db algo direction (:curr-time db))) + ([start-ent-id db algo direction ts] + (let [pend-struct (if (= :bfs algo) vec list*) + explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] + (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) From 4252af34f9fe460277fdf38079dbdb221ea02dd7 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 24 May 2014 10:15:09 +0300 Subject: [PATCH 096/102] typos and spaces based on @beloglazov comments --- functionalDB/src/core/constructs.clj | 2 +- functionalDB/src/core/fdb.clj | 524 +++++++++++++-------------- 2 files changed, 263 insertions(+), 263 deletions(-) diff --git a/functionalDB/src/core/constructs.clj b/functionalDB/src/core/constructs.clj index 02639e3e0..be8a4fb03 100644 --- a/functionalDB/src/core/constructs.clj +++ b/functionalDB/src/core/constructs.clj @@ -28,7 +28,7 @@ [] (atom (Database. [(Timestamped. (initial-storage) ; storage - (make-index #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph qeries and joins + (make-index #(vector %3 %2 %1) #(vector %3 %2 %1) #(ref? %) 0) ; VAET - for graph queries and joins (make-index #(vector %2 %3 %1) #(vector %3 %1 %2) #(not (not %)) 0) ; AVET - for filtering (make-index #(vector %3 %1 %2) #(vector %2 %3 %1) #(not (not %)) 1) ; VEAT - for filtering (make-index #(vector %1 %2 %3) #(vector %1 %2 %3) #(not (not %)) 2) )] ; EAVT - for filtering diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 941b79ce9..450fbc29b 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -1,262 +1,262 @@ -(ns core.fdb - [:use [core storage query constructs]] - [:require [clojure.set :as CS :only (union difference intersection)]]) - -(defn collify [x] (if (coll? x) x [x])) - -(defn next-ts [db] (inc (:curr-time db))) - -(defn next-id - "returns a pair composed of the id to use for the given entity and the next free running id in the database" - [db ent] - (let [top-id (:top-id db) - ent-id (:id ent) - inceased-id (inc top-id)] - (if (= ent-id :db/no-id-yet) - [(keyword (str inceased-id)) inceased-id] - [ent-id top-id]))) - -(defn entity-at - "the entity with the given ent-id at the given time (defualts to the latest time)" - ([db ent-id] (entity-at db (:curr-time db) ent-id)) - ([db ts ent-id] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) - -(defn attr-at - "The attribute of an entity at a given time (defaults to recent time)" - ([db ent-id attr-name] - (attr-at db ent-id attr-name (:curr-time db))) - ([db ent-id attr-name ts] - (get-in (entity-at db ts ent-id) [:attrs attr-name]))) - -(defn value-of-at - "value of a datom at a given time, if no time is provided, we default to the most recent value" - ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) - ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) - -(defn update-attr-value - "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" - [attr value operation] - (cond - (single? attr) (assoc attr :value #{value}) - ; now we're talking about an attribute of multiple values - (= :db/reset-to operation) (assoc attr :value value) - (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) - (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) - -(defn add-attr - "adds an attribute to an entity" - [ent attr] - (let [attr-id (keyword (:name attr))] - (assoc-in ent [:attrs attr-id] attr))) - -(defn update-creation-ts - "updates the timestamp value of all the attributes of an entity to the given timestamp" - [ent ts-val] - (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) - -(defn add-entry-to-index [index path operation] - (if (= operation :db/remove) - index - (let [update-path (butlast path) - update-value (last path) - to-be-updated-set (get-in index update-path #{})] - (assoc-in index update-path (conj to-be-updated-set update-value) )))) - -(defn update-attr-in-index [index ent-id attr-name target-val operation] - (let [ colled-target-val (collify target-val) - add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] - (reduce add-entry-fn index colled-target-val))) - -(defn add-entity-to-index [ent timestamped ind-name] - (let [ent-id (:id ent) - index (ind-name timestamped) - all-attrs (vals (:attrs ent)) - relevant-attrs (filter #((usage-pred index) %) all-attrs ) - add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] - (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) - -(defn fix-new-entity [db ent] - (let [[ent-id next-top-id] (next-id db ent) - new-ts (next-ts db)] - [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) - -(defn add-entity [db ent] - (let [[fixed-ent next-top-id](fix-new-entity db ent) - new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) - add-fn (partial add-entity-to-index fixed-ent) - new-timestamped (reduce add-fn new-timestamped (indices))] - (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) - -(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) - -(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) - -(defn update-attr [attr new-val new-ts operation] - {:pre [(if (single? attr) - (contains? #{:db/reset-to :db/remove} operation) - (contains? #{:db/reset-to :db/add :db/remove} operation))]} - (-> attr - (update-attr-modification-time new-ts) - (update-attr-value new-val operation))) - -(defn remove-path-values [old-vals target-val operation] - (case operation - :db/add [] ; nothing to remove - :db/reset-to old-vals ; removing all of the old values - :db/remove (collify target-val))) ; removing the values defined by the caller - -(defn remove-entry-from-index [index path] - (let [path-head (first path) - path-to-items (butlast path) - val-to-remove (last path) - old-entries-set (get-in index path-to-items)] - (cond - (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here - (and (= 1 (count old-entries-set) ) (= 1 (count (index path-head)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely - (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it - :else (update-in index path-to-items disj val-to-remove)))) - -(defn remove-entries-from-index [ent-id operation index attr] - (if (= operation :db/add) - index - (let [attr-name (:name attr) - datom-vals (collify (:value attr)) - paths (map #((from-eav index) ent-id attr-name %) datom-vals)] - (reduce remove-entry-from-index index paths)))) - -(defn update-index [ent-id old-attr target-val operation timestamped ind-name] - (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) - timestamped - (let [index (ind-name timestamped) - old-vals (:value old-attr) - attr-name (:name old-attr) - remove-paths (remove-path-values old-vals target-val operation) - cleaned-index (remove-entries-from-index ent-id operation index old-attr) - updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] - (assoc timestamped ind-name updated-index)))) - -(defn update-entity [storage e-id new-attr] - (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) - -(defn update-timestamped - [timestamped ent-id old-attr updated-attr new-val operation] - (let [storage (:storage timestamped) - new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] - (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) - -(defn update-datom - ([db ent-id attr-name new-val] - (update-datom db ent-id attr-name new-val :db/reset-to )) - ([db ent-id attr-name new-val operation ] - (let [update-ts (next-ts db) - timestamped (last (:timestamped db)) - attr (attr-at db ent-id attr-name) - updated-attr (update-attr attr new-val update-ts operation) - fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] - (update-in db [:timestamped] conj fully-updated-timestamped)))) - -(defn remove-entity-from-index [ent timestamped ind-name] - (let [ent-id (:id ent) - index (ind-name timestamped) - all-attrs (vals (:attrs ent)) - relevant-attrs (filter #((usage-pred index) %) all-attrs ) - remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] - (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) - -(defn reffing-datoms-to [e-id timestamped] - (let [vaet (:VAET timestamped)] - (for [[attr-name reffing-set] (e-id vaet) - reffing reffing-set] - [reffing attr-name e-id]))) - -(defn remove-back-refs [db e-id timestamped] - (let [refing-datoms (reffing-datoms-to e-id timestamped) - remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) - clean-db (reduce remove-fn db refing-datoms)] - (last (:timestamped db)))) - -(defn remove-entity [db ent-id] - (let [ent (entity-at db ent-id) - timestamped (remove-back-refs db ent-id (last (:timestamped db))) - retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) - no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) - new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] - (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) - -(defn transact-on-db [initial-db txs] - (loop [[tx & rst-tx] txs transacted initial-db] - (if tx - (recur rst-tx (apply (first tx) transacted (rest tx))) - (let [initial-timestamped (:timestamped initial-db) - new-timestamped (last (:timestamped transacted))] - (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) - -(defmacro _transact [db op & txs] - (when txs - (loop [[frst-tx# & rst-tx#] txs res# [op db `transact-on-db] accum-txs# []] - (if frst-tx# - (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) - (list* (conj res# accum-txs#)))))) - -(defn _what-if - "Operates on the db with the given transactions, but without eventually updating it" - [ db f txs] - (f db txs)) - -(defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) - -(defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) - -(defmacro q - "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). - At the moment support only filtering queries, no joins is also assumed." - [db query] - `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query - ind# (choose-index ~db query#) ; selecting which index to use - q-res# (query-index ind# query#) ; actually quering, from now on just preparing results - binded-res# (bind-variables-to-query q-res# ind#) ; building a meta + real results structure - needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables - (map (partial locate-vars-in-query-res needed-vars# ) binded-res#))) ; picking the needed variables from the query result - -(defn evolution-of - "The sequence of the values of of an entity's attribute, as changed through time" - [db ent-id attr-name] - (loop [res [] ts (:curr-time db)] - (if (= -1 ts) (reverse res) - (let [attr (attr-at db ent-id attr-name ts)] - (recur (conj res {(:ts attr) (:value attr)}) (:prev-ts attr)))))) - -(defn db-before - "How the db was before a given timestamp" - [db ts] - (let [timestamped-before (subvec (:timestamped db) 0 ts )] - (assoc db :timestamped timestamped-before :curr-time ts))) - -(defn outgoing-refs [db ts ent-id] - (if ent-id - (->> (entity-at db ts ent-id) - (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) - [])) - -(defn incoming-refs [db ts ent-id] - (let [vaet (ind-at db ts :VAET)] - (reduce into (vals (ent-id vaet))))) - -(defn remove-explored [pendings explored restruct-fn] - (if (contains? explored (first pendings)) - (recur (rest pendings) explored restruct-fn) - (restruct-fn pendings))) - -(defn traverse [pendings explored out-reffing ent-at restruct-fn] - (let [cleaned-pendings (remove-explored pendings explored restruct-fn) - item (first cleaned-pendings) - next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item))] - (when item (cons (ent-at item) - (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at restruct-fn)))))) - -(defn traverse-db - ([start-ent-id db algo direction] (traverse-db start-ent-id db algo direction (:curr-time db))) - ([start-ent-id db algo direction ts] - (let [pend-struct (if (= :bfs algo) vec list*) - explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] - (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) +(ns core.fdb + [:use [core storage query constructs]] + [:require [clojure.set :as CS :only (union difference intersection)]]) + +(defn collify [x] (if (coll? x) x [x])) + +(defn next-ts [db] (inc (:curr-time db))) + +(defn next-id + "returns a pair composed of the id to use for the given entity and the next free running id in the database" + [db ent] + (let [top-id (:top-id db) + ent-id (:id ent) + inceased-id (inc top-id)] + (if (= ent-id :db/no-id-yet) + [(keyword (str inceased-id)) inceased-id] + [ent-id top-id]))) + +(defn entity-at + "the entity with the given ent-id at the given time (defaults to the latest time)" + ([db ent-id] (entity-at db (:curr-time db) ent-id)) + ([db ts ent-id] (stored-entity (get-in db [:timestamped ts :storage]) ent-id))) + +(defn attr-at + "The attribute of an entity at a given time (defaults to recent time)" + ([db ent-id attr-name] + (attr-at db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name ts] + (get-in (entity-at db ts ent-id) [:attrs attr-name]))) + +(defn value-of-at + "value of a datom at a given time, if no time is provided, we default to the most recent value" + ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) + ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) + +(defn update-attr-value + "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" + [attr value operation] + (cond + (single? attr) (assoc attr :value #{value}) + ; now we're talking about an attribute of multiple values + (= :db/reset-to operation) (assoc attr :value value) + (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) + (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) + +(defn add-attr + "adds an attribute to an entity" + [ent attr] + (let [attr-id (keyword (:name attr))] + (assoc-in ent [:attrs attr-id] attr))) + +(defn update-creation-ts + "updates the timestamp value of all the attributes of an entity to the given timestamp" + [ent ts-val] + (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) + +(defn add-entry-to-index [index path operation] + (if (= operation :db/remove) + index + (let [update-path (butlast path) + update-value (last path) + to-be-updated-set (get-in index update-path #{})] + (assoc-in index update-path (conj to-be-updated-set update-value) )))) + +(defn update-attr-in-index [index ent-id attr-name target-val operation] + (let [ colled-target-val (collify target-val) + add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] + (reduce add-entry-fn index colled-target-val))) + +(defn add-entity-to-index [ent timestamped ind-name] + (let [ent-id (:id ent) + index (ind-name timestamped) + all-attrs (vals (:attrs ent)) + relevant-attrs (filter #((usage-pred index) %) all-attrs ) + add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] + (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) + +(defn fix-new-entity [db ent] + (let [[ent-id next-top-id] (next-id db ent) + new-ts (next-ts db)] + [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) + +(defn add-entity [db ent] + (let [[fixed-ent next-top-id](fix-new-entity db ent) + new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) + add-fn (partial add-entity-to-index fixed-ent) + new-timestamped (reduce add-fn new-timestamped (indices))] + (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) + +(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) + +(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) + +(defn update-attr [attr new-val new-ts operation] + {:pre [(if (single? attr) + (contains? #{:db/reset-to :db/remove} operation) + (contains? #{:db/reset-to :db/add :db/remove} operation))]} + (-> attr + (update-attr-modification-time new-ts) + (update-attr-value new-val operation))) + +(defn remove-path-values [old-vals target-val operation] + (case operation + :db/add [] ; nothing to remove + :db/reset-to old-vals ; removing all of the old values + :db/remove (collify target-val))) ; removing the values defined by the caller + +(defn remove-entry-from-index [index path] + (let [path-head (first path) + path-to-items (butlast path) + val-to-remove (last path) + old-entries-set (get-in index path-to-items)] + (cond + (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here + (and (= 1 (count old-entries-set) ) (= 1 (count (index path-head)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely + (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it + :else (update-in index path-to-items disj val-to-remove)))) + +(defn remove-entries-from-index [ent-id operation index attr] + (if (= operation :db/add) + index + (let [attr-name (:name attr) + datom-vals (collify (:value attr)) + paths (map #((from-eav index) ent-id attr-name %) datom-vals)] + (reduce remove-entry-from-index index paths)))) + +(defn update-index [ent-id old-attr target-val operation timestamped ind-name] + (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) + timestamped + (let [index (ind-name timestamped) + old-vals (:value old-attr) + attr-name (:name old-attr) + remove-paths (remove-path-values old-vals target-val operation) + cleaned-index (remove-entries-from-index ent-id operation index old-attr) + updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] + (assoc timestamped ind-name updated-index)))) + +(defn update-entity [storage e-id new-attr] + (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) + +(defn update-timestamped + [timestamped ent-id old-attr updated-attr new-val operation] + (let [storage (:storage timestamped) + new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] + (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) + +(defn update-datom + ([db ent-id attr-name new-val] + (update-datom db ent-id attr-name new-val :db/reset-to )) + ([db ent-id attr-name new-val operation ] + (let [update-ts (next-ts db) + timestamped (last (:timestamped db)) + attr (attr-at db ent-id attr-name) + updated-attr (update-attr attr new-val update-ts operation) + fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] + (update-in db [:timestamped] conj fully-updated-timestamped)))) + +(defn remove-entity-from-index [ent timestamped ind-name] + (let [ent-id (:id ent) + index (ind-name timestamped) + all-attrs (vals (:attrs ent)) + relevant-attrs (filter #((usage-pred index) %) all-attrs ) + remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] + (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) + +(defn reffing-datoms-to [e-id timestamped] + (let [vaet (:VAET timestamped)] + (for [[attr-name reffing-set] (e-id vaet) + reffing reffing-set] + [reffing attr-name e-id]))) + +(defn remove-back-refs [db e-id timestamped] + (let [refing-datoms (reffing-datoms-to e-id timestamped) + remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) + clean-db (reduce remove-fn db refing-datoms)] + (last (:timestamped db)))) + +(defn remove-entity [db ent-id] + (let [ent (entity-at db ent-id) + timestamped (remove-back-refs db ent-id (last (:timestamped db))) + retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) + no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) + new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] + (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) + +(defn transact-on-db [initial-db txs] + (loop [[tx & rst-tx] txs transacted initial-db] + (if tx + (recur rst-tx (apply (first tx) transacted (rest tx))) + (let [initial-timestamped (:timestamped initial-db) + new-timestamped (last (:timestamped transacted))] + (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) + +(defmacro _transact [db op & txs] + (when txs + (loop [[frst-tx# & rst-tx#] txs res# [op db `transact-on-db] accum-txs# []] + (if frst-tx# + (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) + (list* (conj res# accum-txs#)))))) + +(defn _what-if + "Operates on the db with the given transactions, but without eventually updating it" + [ db f txs] + (f db txs)) + +(defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) + +(defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) + +(defmacro q + "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). + At the moment support only filtering queries, no joins is also assumed." + [db query] + `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query + ind# (choose-index ~db query#) ; selecting which index to use + q-res# (query-index ind# query#) ; actually quering, from now on just preparing results + binded-res# (bind-variables-to-query q-res# ind#) ; building a meta + real results structure + needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables + (map (partial locate-vars-in-query-res needed-vars# ) binded-res#))) ; picking the needed variables from the query result + +(defn evolution-of + "The sequence of the values of an entity's attribute, as changed through time" + [db ent-id attr-name] + (loop [res [] ts (:curr-time db)] + (if (= -1 ts) (reverse res) + (let [attr (attr-at db ent-id attr-name ts)] + (recur (conj res {(:ts attr) (:value attr)}) (:prev-ts attr)))))) + +(defn db-before + "How the db was before a given timestamp" + [db ts] + (let [timestamped-before (subvec (:timestamped db) 0 ts )] + (assoc db :timestamped timestamped-before :curr-time ts))) + +(defn outgoing-refs [db ts ent-id] + (if ent-id + (->> (entity-at db ts ent-id) + (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) + [])) + +(defn incoming-refs [db ts ent-id] + (let [vaet (ind-at db ts :VAET)] + (reduce into (vals (ent-id vaet))))) + +(defn remove-explored [pendings explored restruct-fn] + (if (contains? explored (first pendings)) + (recur (rest pendings) explored restruct-fn) + (restruct-fn pendings))) + +(defn traverse [pendings explored out-reffing ent-at restruct-fn] + (let [cleaned-pendings (remove-explored pendings explored restruct-fn) + item (first cleaned-pendings) + next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item))] + (when item (cons (ent-at item) + (lazy-seq (traverse next-pends (conj explored item) out-reffing ent-at restruct-fn)))))) + +(defn traverse-db + ([start-ent-id db algo direction] (traverse-db start-ent-id db algo direction (:curr-time db))) + ([start-ent-id db algo direction ts] + (let [pend-struct (if (= :bfs algo) vec list*) + explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] + (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) From 47660ce3622003478985b818bedf96a88d88eff5 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 24 May 2014 12:11:42 +0300 Subject: [PATCH 097/102] renaming force-get to put achieving naming that follow the usual idempotence semantics of PUT in HTTP calls --- functionalDB/src/core/manage.clj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/core/manage.clj index ad32652a5..6cacb58a6 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/core/manage.clj @@ -4,7 +4,7 @@ (def ^:private __ALL-DBS__ (atom {})) -(defn- force-get-db [dbs db-name] (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) +(defn- put-db [dbs db-name] (if (db-name dbs) dbs (assoc dbs db-name (make-db)))) (defn- drop-db [dbs db-name] (dissoc dbs db-name)) @@ -12,7 +12,7 @@ (defn get-db-conn [db-name] (let [stored-db-name (as-db-name db-name)] - (stored-db-name (swap! __ALL-DBS__ force-get-db stored-db-name)))) + (stored-db-name (swap! __ALL-DBS__ put-db stored-db-name)))) (defn drop-db-conn [db-name] (let [stored-db-name (as-db-name db-name)] From 12ec63b1c477aa95709112f294aa0bdb58f8072b Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 24 May 2014 12:31:47 +0300 Subject: [PATCH 098/102] alignment, spaces and privitizing some functions --- functionalDB/src/core/fdb.clj | 125 +++++++++++++++++----------------- 1 file changed, 62 insertions(+), 63 deletions(-) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/core/fdb.clj index 450fbc29b..6e4bae42e 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/core/fdb.clj @@ -2,11 +2,11 @@ [:use [core storage query constructs]] [:require [clojure.set :as CS :only (union difference intersection)]]) -(defn collify [x] (if (coll? x) x [x])) +(defn- collify [x] (if (coll? x) x [x])) -(defn next-ts [db] (inc (:curr-time db))) +(defn- next-ts [db] (inc (:curr-time db))) -(defn next-id +(defn- next-id "returns a pair composed of the id to use for the given entity and the next free running id in the database" [db ent] (let [top-id (:top-id db) @@ -23,17 +23,16 @@ (defn attr-at "The attribute of an entity at a given time (defaults to recent time)" - ([db ent-id attr-name] - (attr-at db ent-id attr-name (:curr-time db))) + ([db ent-id attr-name] (attr-at db ent-id attr-name (:curr-time db))) ([db ent-id attr-name ts] - (get-in (entity-at db ts ent-id) [:attrs attr-name]))) + (get-in (entity-at db ts ent-id) [:attrs attr-name]))) (defn value-of-at "value of a datom at a given time, if no time is provided, we default to the most recent value" ([db ent-id attr-name] (:value (attr-at db ent-id attr-name))) ([db ent-id attr-name ts] (:value (attr-at db ent-id attr-name ts)))) -(defn update-attr-value +(defn- update-attr-value "updating the attribute value based on the kind of the operation, the cardinality defined for this attribute and the given value" [attr value operation] (cond @@ -43,31 +42,31 @@ (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) -(defn add-attr +(defn- add-attr "adds an attribute to an entity" [ent attr] (let [attr-id (keyword (:name attr))] (assoc-in ent [:attrs attr-id] attr))) -(defn update-creation-ts +(defn- update-creation-ts "updates the timestamp value of all the attributes of an entity to the given timestamp" [ent ts-val] (reduce #(assoc-in %1 [:attrs %2 :ts ] ts-val) ent (keys (:attrs ent)))) -(defn add-entry-to-index [index path operation] +(defn- add-entry-to-index [index path operation] (if (= operation :db/remove) index (let [update-path (butlast path) - update-value (last path) - to-be-updated-set (get-in index update-path #{})] + update-value (last path) + to-be-updated-set (get-in index update-path #{})] (assoc-in index update-path (conj to-be-updated-set update-value) )))) -(defn update-attr-in-index [index ent-id attr-name target-val operation] - (let [ colled-target-val (collify target-val) - add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] +(defn- update-attr-in-index [index ent-id attr-name target-val operation] + (let [colled-target-val (collify target-val) + add-entry-fn (fn [indx vl] (add-entry-to-index indx ((from-eav index) ent-id attr-name vl) operation))] (reduce add-entry-fn index colled-target-val))) -(defn add-entity-to-index [ent timestamped ind-name] +(defn- add-entity-to-index [ent timestamped ind-name] (let [ent-id (:id ent) index (ind-name timestamped) all-attrs (vals (:attrs ent)) @@ -75,23 +74,23 @@ add-in-index-fn (fn [ind attr] (update-attr-in-index ind ent-id (:name attr) (:value attr) :db/add))] (assoc timestamped ind-name (reduce add-in-index-fn index relevant-attrs)))) -(defn fix-new-entity [db ent] +(defn- fix-new-entity [db ent] (let [[ent-id next-top-id] (next-id db ent) - new-ts (next-ts db)] + new-ts (next-ts db)] [(update-creation-ts (assoc ent :id ent-id) new-ts) next-top-id])) -(defn add-entity [db ent] - (let [[fixed-ent next-top-id](fix-new-entity db ent) - new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) - add-fn (partial add-entity-to-index fixed-ent) - new-timestamped (reduce add-fn new-timestamped (indices))] +(defn add-entity [db ent] + (let [[fixed-ent next-top-id] (fix-new-entity db ent) + new-timestamped (update-in (last (:timestamped db)) [:storage] update-storage fixed-ent) + add-fn (partial add-entity-to-index fixed-ent) + new-timestamped (reduce add-fn new-timestamped (indices))] (assoc db :timestamped (conj (:timestamped db) new-timestamped) :top-id next-top-id))) -(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) +(defn add-entities [db ents-seq] (reduce add-entity db ents-seq)) -(defn update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) +(defn- update-attr-modification-time [attr new-ts] (assoc attr :ts new-ts :prev-ts ( :ts attr))) -(defn update-attr [attr new-val new-ts operation] +(defn- update-attr [attr new-val new-ts operation] {:pre [(if (single? attr) (contains? #{:db/reset-to :db/remove} operation) (contains? #{:db/reset-to :db/add :db/remove} operation))]} @@ -99,32 +98,32 @@ (update-attr-modification-time new-ts) (update-attr-value new-val operation))) -(defn remove-path-values [old-vals target-val operation] +(defn- remove-path-values [old-vals target-val operation] (case operation :db/add [] ; nothing to remove :db/reset-to old-vals ; removing all of the old values :db/remove (collify target-val))) ; removing the values defined by the caller -(defn remove-entry-from-index [index path] +(defn- remove-entry-from-index [index path] (let [path-head (first path) - path-to-items (butlast path) - val-to-remove (last path) - old-entries-set (get-in index path-to-items)] + path-to-items (butlast path) + val-to-remove (last path) + old-entries-set (get-in index path-to-items)] (cond (not (contains? old-entries-set val-to-remove)) index ; the set of items does not contain the item to remove, => nothing to do here (and (= 1 (count old-entries-set) ) (= 1 (count (index path-head)))) (dissoc index path-head) ; a path representing a single EAV - remove it entirely (= (count old-entries-set) 1) (update-in index [path-head] dissoc (second path)) ; a path that splits at the second item - just remove the unneeded part of it :else (update-in index path-to-items disj val-to-remove)))) -(defn remove-entries-from-index [ent-id operation index attr] +(defn- remove-entries-from-index [ent-id operation index attr] (if (= operation :db/add) index (let [attr-name (:name attr) - datom-vals (collify (:value attr)) - paths (map #((from-eav index) ent-id attr-name %) datom-vals)] + datom-vals (collify (:value attr)) + paths (map #((from-eav index) ent-id attr-name %) datom-vals)] (reduce remove-entry-from-index index paths)))) -(defn update-index [ent-id old-attr target-val operation timestamped ind-name] +(defn- update-index [ent-id old-attr target-val operation timestamped ind-name] (if-not ((usage-pred (get-in timestamped [ind-name])) old-attr) timestamped (let [index (ind-name timestamped) @@ -135,13 +134,13 @@ updated-index (update-attr-in-index cleaned-index ent-id attr-name target-val operation)] (assoc timestamped ind-name updated-index)))) -(defn update-entity [storage e-id new-attr] +(defn- update-entity [storage e-id new-attr] (assoc-in (stored-entity storage e-id) [:attrs (:name new-attr)] new-attr)) -(defn update-timestamped +(defn- update-timestamped [timestamped ent-id old-attr updated-attr new-val operation] (let [storage (:storage timestamped) - new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] + new-timestamped (reduce (partial update-index ent-id old-attr new-val operation) timestamped (indices))] (assoc new-timestamped :storage (update-storage storage (update-entity storage ent-id updated-attr))))) (defn update-datom @@ -149,13 +148,13 @@ (update-datom db ent-id attr-name new-val :db/reset-to )) ([db ent-id attr-name new-val operation ] (let [update-ts (next-ts db) - timestamped (last (:timestamped db)) - attr (attr-at db ent-id attr-name) - updated-attr (update-attr attr new-val update-ts operation) - fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] + timestamped (last (:timestamped db)) + attr (attr-at db ent-id attr-name) + updated-attr (update-attr attr new-val update-ts operation) + fully-updated-timestamped (update-timestamped timestamped ent-id attr updated-attr new-val operation)] (update-in db [:timestamped] conj fully-updated-timestamped)))) -(defn remove-entity-from-index [ent timestamped ind-name] +(defn- remove-entity-from-index [ent timestamped ind-name] (let [ent-id (:id ent) index (ind-name timestamped) all-attrs (vals (:attrs ent)) @@ -163,24 +162,24 @@ remove-from-index-fn (partial remove-entries-from-index ent-id :db/remove)] (assoc timestamped ind-name (reduce remove-from-index-fn index relevant-attrs)))) -(defn reffing-datoms-to [e-id timestamped] +(defn- reffing-datoms-to [e-id timestamped] (let [vaet (:VAET timestamped)] (for [[attr-name reffing-set] (e-id vaet) - reffing reffing-set] - [reffing attr-name e-id]))) + reffing reffing-set] + [reffing attr-name e-id]))) -(defn remove-back-refs [db e-id timestamped] +(defn- remove-back-refs [db e-id timestamped] (let [refing-datoms (reffing-datoms-to e-id timestamped) - remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) - clean-db (reduce remove-fn db refing-datoms)] - (last (:timestamped db)))) + remove-fn (fn[d [e a v]] (update-datom db e a v :db/remove)) + clean-db (reduce remove-fn db refing-datoms)] + (last (:timestamped db)))) -(defn remove-entity [db ent-id] +(defn remove-entity [db ent-id] (let [ent (entity-at db ent-id) - timestamped (remove-back-refs db ent-id (last (:timestamped db))) - retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) - no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) - new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] + timestamped (remove-back-refs db ent-id (last (:timestamped db))) + retimed-timestamped (update-in timestamped [:VAET] dissoc ent-id) + no-ent-timestamped (assoc retimed-timestamped :storage (remove-entity-from-storage (:storage retimed-timestamped) ent)) + new-timestamped (reduce (partial remove-entity-from-index ent) no-ent-timestamped (indices))] (assoc db :timestamped (conj (:timestamped db) new-timestamped)))) (defn transact-on-db [initial-db txs] @@ -188,8 +187,8 @@ (if tx (recur rst-tx (apply (first tx) transacted (rest tx))) (let [initial-timestamped (:timestamped initial-db) - new-timestamped (last (:timestamped transacted))] - (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) + new-timestamped (last (:timestamped transacted))] + (assoc initial-db :timestamped (conj initial-timestamped new-timestamped) :curr-time (next-ts initial-db) :top-id (:top-id transacted)))))) (defmacro _transact [db op & txs] (when txs @@ -235,19 +234,19 @@ (defn outgoing-refs [db ts ent-id] (if ent-id (->> (entity-at db ts ent-id) - (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) + (:attrs) (vals) (filter ref?) (mapcat (comp collify :value))) [])) -(defn incoming-refs [db ts ent-id] +(defn- incoming-refs [db ts ent-id] (let [vaet (ind-at db ts :VAET)] (reduce into (vals (ent-id vaet))))) -(defn remove-explored [pendings explored restruct-fn] +(defn- remove-explored [pendings explored restruct-fn] (if (contains? explored (first pendings)) (recur (rest pendings) explored restruct-fn) (restruct-fn pendings))) -(defn traverse [pendings explored out-reffing ent-at restruct-fn] +(defn- traverse [pendings explored out-reffing ent-at restruct-fn] (let [cleaned-pendings (remove-explored pendings explored restruct-fn) item (first cleaned-pendings) next-pends (reduce conj (restruct-fn (rest cleaned-pendings)) (out-reffing item))] @@ -258,5 +257,5 @@ ([start-ent-id db algo direction] (traverse-db start-ent-id db algo direction (:curr-time db))) ([start-ent-id db algo direction ts] (let [pend-struct (if (= :bfs algo) vec list*) - explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] - (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) + explor-fn (if (= :outgoing direction) outgoing-refs incoming-refs)] + (traverse [start-ent-id] #{} (partial explor-fn db ts) (partial entity-at db ts) pend-struct)))) From b93f7ae657b26078925c6856b5238a0f14473eca Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 24 May 2014 12:35:48 +0300 Subject: [PATCH 099/102] alignment --- functionalDB/src/core/query.clj | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/core/query.clj index 3bf8d9dfc..e2f47a517 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/core/query.clj @@ -4,7 +4,7 @@ (defn ind-at "inspecting a specific index at a given time, defaults to current. The kind argument mayone of the index name (e.g., AVET)" - ([db kind] + ([db kind] (ind-at db kind (:curr-time db))) ([db kind ts] (kind ((:timestamped db) ts)))) @@ -62,7 +62,7 @@ a vector in which the first element is the decided index and the second element is a function that knows how to restore an EAV structure from that decided index path structure." [db query] (let [var-ind (index-of-chaining-variable query) - ind-to-use (case var-ind 0 :AVET 1 :VEAT 2 :EAVT)] + ind-to-use (case var-ind 0 :AVET 1 :VEAT 2 :EAVT)] (ind-at db ind-to-use))) (defn filter-index @@ -71,12 +71,12 @@ [index path-preds] (for [ path-pred path-preds :let [[lvl1-prd lvl2-prd lvl3-prd] (apply (from-eav index) path-pred)] ; predicates for the first and second level of the index, also keeping the path to later use its meta - [k1 l2map] index ; keys and values of the first level - :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate - [k2 l3-set] l2map ; keys and values of the second level - :when (try (lvl2-prd k2) (catch Exception e false)) ; filtering to keep only the keys and vals of keys that passed the second level predicate - :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them - (with-meta [k1 k2 res] (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables + [k1 l2map] index ; keys and values of the first level + :when (try (lvl1-prd k1) (catch Exception e false)) ; filtering to keep only the keys and the vals of the keys that passed the first level predicate + [k2 l3-set] l2map ; keys and values of the second level + :when (try (lvl2-prd k2) (catch Exception e false)) ; filtering to keep only the keys and vals of keys that passed the second level predicate + :let [res (set (filter lvl3-prd l3-set))] ]; keep from the set at the third level only the items that passed the predicate on them + (with-meta [k1 k2 res] (meta path-pred)))) ; constructed to resemble the EAV structure, while keeping the meta of the query to use it later when extracting variables (defn items-that-answer-all-conditions "takes the sequence of all the items collection, each such collection answered one condition, we test here what are the items that answered all of the conditions @@ -100,7 +100,7 @@ returns for a result path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item is followed by its variable name as was inserted in the query" [index path] - (let [seq-path [ (repeat (first path)) (repeat (second path)) (last path)] + (let [seq-path [(repeat (first path)) (repeat (second path)) (last path)] meta-path(apply (from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index all-path (interleave meta-path seq-path)] (apply (partial map vector) all-path))) @@ -111,7 +111,7 @@ attribute, and the value is the binding pair of that found attribute's value" [q-res index] (let [seq-res-path (mapcat (partial seqify-index-path index) q-res) ; seq-ing a result to hold the meta - res-path (map #(->> %1 (partition 2)(apply (to-eav index))) seq-res-path)] ; making binding pairs + res-path (map #(->> %1 (partition 2)(apply (to-eav index))) seq-res-path)] ; making binding pairs (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) ; structuring the pairs into the wanted binding structure (defn query-index @@ -140,7 +140,7 @@ "this function would look for all the binding found in the query result and return the binding that were requested by the user (captured at the vars-set)" [vars-set q-res] (let [[e-pair av-map] q-res - e-res (resultify-bind-pair vars-set [] e-pair )] + e-res (resultify-bind-pair vars-set [] e-pair )] (reduce (partial resultify-av-pair vars-set) e-res av-map))) (defmacro settify [coll] (set (map str coll))) From 92f922f25232daded25b4e70dd9226ca92bb9595 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 24 May 2014 13:27:04 +0300 Subject: [PATCH 100/102] renaming core folder to be fdb --- functionalDB/src/{core => fdb}/constructs.clj | 4 ++-- functionalDB/src/{core/fdb.clj => fdb/core.clj} | 6 +++--- functionalDB/src/{core => fdb}/manage.clj | 6 +++--- functionalDB/src/{core => fdb}/query.clj | 4 ++-- functionalDB/src/{core => fdb}/storage.clj | 2 +- functionalDB/src/scenarios/hospital.clj | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) rename functionalDB/src/{core => fdb}/constructs.clj (98%) rename functionalDB/src/{core/fdb.clj => fdb/core.clj} (99%) rename functionalDB/src/{core => fdb}/manage.clj (88%) rename functionalDB/src/{core => fdb}/query.clj (99%) rename functionalDB/src/{core => fdb}/storage.clj (93%) diff --git a/functionalDB/src/core/constructs.clj b/functionalDB/src/fdb/constructs.clj similarity index 98% rename from functionalDB/src/core/constructs.clj rename to functionalDB/src/fdb/constructs.clj index be8a4fb03..e53081aa6 100644 --- a/functionalDB/src/core/constructs.clj +++ b/functionalDB/src/fdb/constructs.clj @@ -1,5 +1,5 @@ -(ns core.constructs - (:use core.storage)) +(ns fdb.constructs + (:use fdb.storage)) (defrecord Database [timestamped top-id curr-time]) (defrecord Timestamped [storage VAET AVET VEAT EAVT]) diff --git a/functionalDB/src/core/fdb.clj b/functionalDB/src/fdb/core.clj similarity index 99% rename from functionalDB/src/core/fdb.clj rename to functionalDB/src/fdb/core.clj index 6e4bae42e..1e6f247d4 100644 --- a/functionalDB/src/core/fdb.clj +++ b/functionalDB/src/fdb/core.clj @@ -1,5 +1,5 @@ -(ns core.fdb - [:use [core storage query constructs]] +(ns fdb.core + [:use [fdb storage query constructs]] [:require [clojure.set :as CS :only (union difference intersection)]]) (defn- collify [x] (if (coll? x) x [x])) @@ -42,7 +42,7 @@ (= :db/add operation) (assoc attr :value (CS/union (:value attr) value)) (= :db/remove operation) (assoc attr :value (CS/difference (:value attr) value)))) -(defn- add-attr +(defn add-attr "adds an attribute to an entity" [ent attr] (let [attr-id (keyword (:name attr))] diff --git a/functionalDB/src/core/manage.clj b/functionalDB/src/fdb/manage.clj similarity index 88% rename from functionalDB/src/core/manage.clj rename to functionalDB/src/fdb/manage.clj index 6cacb58a6..c3e2de7c5 100644 --- a/functionalDB/src/core/manage.clj +++ b/functionalDB/src/fdb/manage.clj @@ -1,6 +1,6 @@ -(ns core.manage - "Management of db connections (from the user's perspective), internally creates/ / drops dbs." - (:use core.constructs)) +(ns fdb.manage + "Management of db connections (from the user's perspective), internally creates / drops dbs." + (:use fdb.constructs)) (def ^:private __ALL-DBS__ (atom {})) diff --git a/functionalDB/src/core/query.clj b/functionalDB/src/fdb/query.clj similarity index 99% rename from functionalDB/src/core/query.clj rename to functionalDB/src/fdb/query.clj index e2f47a517..6b7be14fb 100644 --- a/functionalDB/src/core/query.clj +++ b/functionalDB/src/fdb/query.clj @@ -1,5 +1,5 @@ -(ns core.query - [:use [core constructs] +(ns fdb.query + [:use [fdb constructs] [clojure.set :as CS :only (intersection)]]) (defn ind-at diff --git a/functionalDB/src/core/storage.clj b/functionalDB/src/fdb/storage.clj similarity index 93% rename from functionalDB/src/core/storage.clj rename to functionalDB/src/fdb/storage.clj index 99f699d62..b79eaba0f 100644 --- a/functionalDB/src/core/storage.clj +++ b/functionalDB/src/fdb/storage.clj @@ -1,4 +1,4 @@ -(ns core.storage) +(ns fdb.storage) (defn initial-storage [] {}) diff --git a/functionalDB/src/scenarios/hospital.clj b/functionalDB/src/scenarios/hospital.clj index e07f6edb4..7c56c7fd8 100644 --- a/functionalDB/src/scenarios/hospital.clj +++ b/functionalDB/src/scenarios/hospital.clj @@ -1,7 +1,7 @@ (ns scenarios.hospital - (:use [core fdb constructs ] ) - [:require [core.manage :as M] - [clojure.set :as CS :only (union difference )]]) + (:use [fdb core constructs ] ) + [:require [fdb.manage :as M] + [clojure.set :as CS :only (union difference )]]) (def db-name "hos12") (M/drop-db-conn db-name) From 9bd3c5335db434bc62fc0d954bbe20315c8a0660 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Sat, 24 May 2014 22:38:53 +0300 Subject: [PATCH 101/102] better naming and clearer comments --- functionalDB/src/fdb/core.clj | 7 +++---- functionalDB/src/fdb/query.clj | 10 +++++----- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/functionalDB/src/fdb/core.clj b/functionalDB/src/fdb/core.clj index 1e6f247d4..d68be5e63 100644 --- a/functionalDB/src/fdb/core.clj +++ b/functionalDB/src/fdb/core.clj @@ -197,10 +197,9 @@ (recur rst-tx# res# (conj accum-txs# (vec frst-tx#))) (list* (conj res# accum-txs#)))))) -(defn _what-if +(defn- _what-if "Operates on the db with the given transactions, but without eventually updating it" - [ db f txs] - (f db txs)) + [db f txs] (f db txs)) (defmacro what-if [db & txs] `(_transact ~db _what-if ~@txs)) @@ -214,7 +213,7 @@ ind# (choose-index ~db query#) ; selecting which index to use q-res# (query-index ind# query#) ; actually quering, from now on just preparing results binded-res# (bind-variables-to-query q-res# ind#) ; building a meta + real results structure - needed-vars# (settify ~(:find query))] ; extracting out the needed to be reported variables + needed-vars# (symbol-col-to-set ~(:find query))] ; extracting out the needed to be reported variables (map (partial locate-vars-in-query-res needed-vars# ) binded-res#))) ; picking the needed variables from the query result (defn evolution-of diff --git a/functionalDB/src/fdb/query.clj b/functionalDB/src/fdb/query.clj index 6b7be14fb..888a33668 100644 --- a/functionalDB/src/fdb/query.clj +++ b/functionalDB/src/fdb/query.clj @@ -106,7 +106,7 @@ (apply (partial map vector) all-path))) (defn bind-variables-to-query - "A function that receives the query results and restructues them to be an binding structure which resembles in its shape to an entity. + "A function that receives the query results and restructues them to be a binding structure which resembles in its shape to an entity. The binding structure is a map whose key is a binding pair of a found entity-id, and the value is also a map, where its key is the binding pair of a found attribute, and the value is the binding pair of that found attribute's value" [q-res index] @@ -116,8 +116,8 @@ (defn query-index "Quering an index based a seq of path predicates. A path predicate is composed of 3 predicates, each one to operate on a different level of the index. Querying an index with - a specific path-pred returns a result-path. We then take all the result paths and find within them the items that passed all the path-preds, and eventually return the result path, each contains - only the items that passed all the path predicates." + a specific path-pred returns a result-path. We then take all the result paths, find within them the last-level-items that are found in all the result-paths, and return the result paths, each contains + only the last-level-items that are part of all the result-paths." [index path-preds] (let [result-paths (filter-index index path-preds) ; the paths (vectors) from the root of the index to the leaves (a leaf of an index is a set) where each path fulfils one predicate path relevant-items (items-that-answer-all-conditions (map last result-paths) (count path-preds)) ; the set of elements, each answers all the pred-paths @@ -137,10 +137,10 @@ (reduce (partial resultify-bind-pair vars-set) accum-res av-pair)) (defn locate-vars-in-query-res - "this function would look for all the binding found in the query result and return the binding that were requested by the user (captured at the vars-set)" + "this function would look for all the bindings found in the query result and return the binding that were requested by the user (captured at the vars-set)" [vars-set q-res] (let [[e-pair av-map] q-res e-res (resultify-bind-pair vars-set [] e-pair )] (reduce (partial resultify-av-pair vars-set) e-res av-map))) -(defmacro settify [coll] (set (map str coll))) +(defmacro symbol-col-to-set [coll] (set (map str coll))) From ebe3131c076920591890e9cc2e9986718cc194f6 Mon Sep 17 00:00:00 2001 From: Yoav Rubin Date: Mon, 26 May 2014 05:48:12 +0300 Subject: [PATCH 102/102] renaming and better docs --- functionalDB/src/fdb/core.clj | 2 +- functionalDB/src/fdb/query.clj | 25 ++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/functionalDB/src/fdb/core.clj b/functionalDB/src/fdb/core.clj index d68be5e63..b9ed8828e 100644 --- a/functionalDB/src/fdb/core.clj +++ b/functionalDB/src/fdb/core.clj @@ -206,7 +206,7 @@ (defmacro transact [db & txs] `(_transact ~db swap! ~@txs)) (defmacro q - "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). + "querying the database using datalog queries built in a map structure ({:find [variables*] :where [ [e a v]* ]}). (after the where there are clauses) At the moment support only filtering queries, no joins is also assumed." [db query] `(let [query# (q-clauses ~(:where query)) ; extracting the query clauses from the query diff --git a/functionalDB/src/fdb/query.clj b/functionalDB/src/fdb/query.clj index 888a33668..63dbdc87d 100644 --- a/functionalDB/src/fdb/query.clj +++ b/functionalDB/src/fdb/query.clj @@ -95,22 +95,21 @@ [relevant-items path] (update-in path [2] CS/intersection relevant-items )) - (defn seqify-index-path - "A result-path is a path whose leaves are the items representing the items of the query chaining variable. This function - returns for a result path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item is followed - by its variable name as was inserted in the query" - [index path] - (let [seq-path [(repeat (first path)) (repeat (second path)) (last path)] - meta-path(apply (from-eav index) (map repeat (:db/variable (meta path)))) ; re-ordering the meta to be in the order of the index - all-path (interleave meta-path seq-path)] - (apply (partial map vector) all-path))) + (defn combine-path-and-meta + "This function returns for a (result) path a seq of vectors, each vector is a path from the root of the result path to one of its items, each item + is followed by its variable name as was inserted in the query (which was kept at the metadata of the (result) path." + [from-eav-fn path] + (let [expanded-path [(repeat (first path)) (repeat (second path)) (last path)] ; there may be several leaves in each path, so repeating the first and second elements + meta-of-path(apply from-eav-fn (map repeat (:db/variable (meta path)))) ; re-ordering the path's meta to be in the order of the index + combined-data-and-meta-path (interleave meta-of-path expanded-path)] + (apply (partial map vector) combined-data-and-meta-path))) ; returning a seq of vectors, each one is a single result with its meta (defn bind-variables-to-query - "A function that receives the query results and restructues them to be a binding structure which resembles in its shape to an entity. - The binding structure is a map whose key is a binding pair of a found entity-id, and the value is also a map, where its key is the binding pair of a found - attribute, and the value is the binding pair of that found attribute's value" + "A function that receives the query results (seq of tripplets) and transforms each of them into a binding structure. + A binding structure is a map whose key is a binding pair of an entity-id, and the value is also a map, where its key is a binding pair + of an attribute, and the value is a binding pair of that found attribute's value. The symbol name in each binding pair is extracted from the trippet's metadata" [q-res index] - (let [seq-res-path (mapcat (partial seqify-index-path index) q-res) ; seq-ing a result to hold the meta + (let [seq-res-path (mapcat (partial combine-path-and-meta (from-eav index)) q-res) ; seq-ing a result to hold the meta res-path (map #(->> %1 (partition 2)(apply (to-eav index))) seq-res-path)] ; making binding pairs (reduce #(assoc-in %1 (butlast %2) (last %2)) {} res-path))) ; structuring the pairs into the wanted binding structure