explain api for explaining query execution

src/fluree/db/api.cljc

@@ -644,6 +644,12 @@
   ([named-graphs default-graphs]
    (query-api/dataset named-graphs default-graphs)))
 
+(defn explain
+  ([ds q]
+   (explain ds q {}))
+  ([ds q opts]
+   (promise-wrap (query-api/explain ds q opts))))
+
 (defn query
   "Executes a query against a database or dataset.
 

src/fluree/db/query/api.cljc

@@ -126,6 +126,10 @@
          (<? (track-execution ds* tracker #(fql/query ds* tracker query**)))
          (<? (fql/query ds* query**)))))))
 
+(defn explain
+  [ds query override-opts]
+  (query-fql ds query (assoc override-opts :meta true)))
+
 (defn query-sparql
   [db query override-opts]
   (go-try

src/fluree/db/query/exec.cljc

@@ -95,9 +95,10 @@
   executor function."
   [q]
   (update q :where #(walk/postwalk (fn [x]
-                                     (if (= :query (where/pattern-type x))
-                                       (let [subquery (second x)]
-                                         (where/->pattern :query (subquery-executor subquery)))
+                                     (if (where/subquery? x)
+                                       (let [subquery (where/pattern-data x)]
+                                         (with-meta (where/->pattern :query (subquery-executor subquery))
+                                           (meta x)))
                                        x))
                                    %)))
 

src/fluree/db/query/exec/where.cljc

@@ -14,8 +14,7 @@
             [fluree.db.util.json :as json]
             [fluree.db.util.log :as log :include-macros true]
             [fluree.db.util.xhttp :as xhttp]
-            [fluree.json-ld :as json-ld])
-  #?(:clj (:import (clojure.lang MapEntry))))
+            [fluree.json-ld :as json-ld]))
 
 #?(:clj (set! *warn-on-reflection* true))
 
@@ -210,11 +209,16 @@
   [graph-alias]
   (str/starts-with? graph-alias "##"))
 
+(defrecord Pattern [pattern-type data])
+
+(defn pattern?
+  [x]
+  (instance? Pattern x))
+
 (defn ->pattern
   "Build a new non-tuple match pattern of type `typ`."
   [typ data]
-  #?(:clj  (MapEntry/create typ data)
-     :cljs (MapEntry. typ data nil)))
+  (->Pattern typ data))
 
 (defn ->iri-ref
   [x]
@@ -316,14 +320,14 @@
 
 (defn pattern-type
   [pattern]
-  (if (map-entry? pattern)
-    (key pattern)
+  (if (pattern? pattern)
+    (:pattern-type pattern)
     :tuple))
 
 (defn pattern-data
   [pattern]
-  (if (map-entry? pattern)
-    (val pattern)
+  (if (pattern? pattern)
+    (:data pattern)
     pattern))
 
 (defmulti match-pattern
@@ -333,6 +337,16 @@
   (fn [_ds _tracker _solution pattern _error-ch]
     (pattern-type pattern)))
 
+(defn match-and-track-pattern
+  [ds tracker solution pattern error-ch]
+  (if (:explain tracker)
+    (do (track/pattern-in! tracker pattern solution)
+        (-> (match-pattern ds tracker solution pattern error-ch)
+            (async/pipe (async/chan 2 (map (fn [solution]
+                                             (track/pattern-out! tracker pattern solution)
+                                             solution))))))
+    (match-pattern ds tracker solution pattern error-ch)))
+
 (defn assign-solution-filter
   [component solution]
   (if (::fn component)
@@ -652,7 +666,7 @@
     (async/pipeline-async 2
                           out-ch
                           (fn [solution ch]
-                            (-> (match-pattern ds tracker solution pattern error-ch)
+                            (-> (match-and-track-pattern ds tracker solution pattern error-ch)
                                 (async/pipe ch)))
                           solution-ch)
     out-ch))
@@ -673,8 +687,7 @@
 
 (defn subquery?
   [pattern]
-  (and (sequential? pattern)
-       (= :query (first pattern))))
+  (= :query (pattern-type pattern)))
 
 (defn match-clause
   "Returns a channel that will eventually contain all match solutions in the

src/fluree/db/query/fql/parse.cljc

@@ -331,6 +331,11 @@
   (fn [pattern _var-config _context]
     (v/where-pattern-type pattern)))
 
+(defn parse-pattern-with-orig
+  "Wrap each parsed pattern with its source so we can map it back for explain queries."
+  [pattern var-config context]
+  (map #(with-meta % {:orig pattern :context context}) (parse-pattern pattern var-config context)))
+
 (defn parse-bind-map
   [binds context]
   (into {}
@@ -353,8 +358,7 @@
                   [clause]
                   (util/sequential clause))]
     (->> clause*
-         (mapcat (fn [pattern]
-                   (parse-pattern pattern var-config context)))
+         (mapcat (fn [pattern] (parse-pattern-with-orig pattern var-config context)))
          where/->where-clause)))
 
 (defn parse-variable-attributes
@@ -627,11 +631,11 @@
   underlying components."
   [patterns]
   (->> patterns
-       (mapcat (fn [[pattern-type component :as pattern]]
+       (mapcat (fn [{:keys [pattern-type data] :as pattern}]
                  (case pattern-type
-                   :class         [component]
-                   :property-join component
-                   :id            [[component]]
+                   :class         [data]
+                   :property-join data
+                   :id            [[data]]
                    [pattern])))
        vec))
 

src/fluree/db/track.cljc

@@ -1,6 +1,7 @@
 (ns fluree.db.track
   (:require [fluree.db.track.fuel :as fuel]
             [fluree.db.track.policy :as policy]
+            [fluree.db.track.solutions :as solutions]
             [fluree.db.track.time :as time]))
 
 #?(:clj (set! *warn-on-reflection* true))
@@ -30,18 +31,25 @@
   (or (track-all? opts)
       (-> meta :policy true?)))
 
+(defn track-solutions?
+  [{:keys [meta] :as opts}]
+  (or (track-all? opts)
+      (:explain meta)))
+
 (defn track-query?
   [opts]
   (or (track-time? opts)
       (track-fuel? opts)
-      (track-policy? opts)))
+      (track-policy? opts)
+      (track-solutions? opts)))
 
 (defn track-txn?
   [opts]
   (or (track-time? opts)
       (track-fuel? opts)
       (track-file? opts)
-      (track-policy? opts)))
+      (track-policy? opts)
+      (track-solutions? opts)))
 
 (defn init-time
   [tracker]
@@ -55,13 +63,28 @@
   [tracker]
   (assoc tracker :policy (policy/init)))
 
+(defn init-explain
+  [tracker]
+  (assoc tracker :explain (solutions/init)))
+
 (defn init
   "Creates a new fuel tracker w/ optional fuel limit (0 means unlimited)."
   [{:keys [max-fuel] :as opts}]
   (cond-> {}
     (track-time? opts) init-time
     (track-fuel? opts) (init-fuel max-fuel)
-    (track-policy? opts) init-policy))
+    (track-policy? opts) init-policy
+    (track-solutions? opts) init-explain))
+
+(defn pattern-in!
+  [tracker pattern solution]
+  (when-let [solution-tracker (:explain tracker)]
+    (solutions/pattern-in! solution-tracker pattern solution)))
+
+(defn pattern-out!
+  [tracker pattern solution]
+  (when-let [solution-tracker (:explain tracker)]
+    (solutions/pattern-out! solution-tracker pattern solution)))
 
 (defn track-fuel!
   [tracker error-ch]
@@ -88,4 +111,5 @@
   (cond-> tracker
     (contains? tracker :time) (update :time time/tally)
     (contains? tracker :fuel) (update :fuel fuel/tally)
-    (contains? tracker :policy) (update :policy policy/tally)))
+    (contains? tracker :policy) (update :policy policy/tally)
+    (contains? tracker :explain) (update :explain solutions/tally)))

src/fluree/db/track/solutions.cljc

@@ -0,0 +1,104 @@
+(ns fluree.db.track.solutions
+  (:require [fluree.db.constants :as const]
+            [fluree.db.query.exec.where :as-alias where]
+            [fluree.json-ld :as json-ld]))
+
+(defn init
+  "Map of `<pattern>->{:in <count> :out <count>}`, where `pattern` is a where-clause pattern,
+  `:in` is the number of solutions the pattern took as input and `:out` is the number of
+  solutions the pattern produced. `:patterns` is the order the patterns were evaluated."
+  []
+  (atom {:patterns []}))
+
+(def initial-stats
+  {:in        0
+   :out       0
+   :binds-in #{}
+   :binds-out #{}})
+
+(defn update-in-stats
+  [{:keys [binds-in] :as stats} solution]
+  (-> stats
+      (update :in inc)
+      (assoc :binds-in (reduce (fn [binds-in* var]
+                                 ;; maintain insert order as metadata
+                                 (conj binds-in* (with-meta var {:ord (count binds-in*)})))
+                               binds-in
+                               (keys solution)))))
+
+(defn pattern-in!
+  "Increment :in counter for pattern."
+  [tracker pattern solution]
+  (swap! tracker
+         (fn [explain]
+           (cond-> (update explain pattern (fnil update-in-stats initial-stats) solution)
+             ;; if pattern isn't tracked yet, add it to :patterns sequence
+             (not (get explain pattern)) (update :patterns conj pattern)))))
+
+(defn update-out-stats
+  [{:keys [binds-in] :as stats} solution]
+  (-> stats
+      (update :out inc)
+      (assoc :binds-out (reduce (fn [binds-out* var]
+                                  ;; maintain insert order as metadata
+                                  (conj binds-out* (with-meta var {:ord (count binds-out*)})))
+                                binds-in
+                                (keys solution)))))
+
+(defn pattern-out!
+  "Increment :out counter for pattern."
+  [tracker pattern solution]
+  (swap! tracker (fn [explain] (update explain pattern update-out-stats solution))))
+
+(defn multi-triple-node-pattern?
+  "When a 'node' pattern has more than two entries and gets parsed to multiple triple patterns."
+  [orig]
+  (and (map? orig) ; node pattern
+       (> (count orig) 2))) ; more than two entries => more than one triple pattern
+
+(defn display-triple-pattern
+  "Users can express a series of triple patterns in a single 'node' pattern.
+
+  e.g. where clause:
+  [{:id ?s :ex/foo ?foo :ex/bar ?bar}]
+  maps to:
+  [ [{::where/var ?s} {::where/iri \"ex:foo\"} {::where/var ?foo}]
+    [{::where/var ?s} {::where/iri \"ex:bar\"} {::where/var ?bar}] ]
+
+  By using the supplied context we can map the expanded subject id and predicate iri
+  back to the user's corresponding syntax.
+
+  [ [{:id ?s :ex/foo ?foo} <counters>]
+    [{:id ?s :ex/bar ?bar} <counters>] ]"
+  [pattern context orig]
+  (let [[_ p _] pattern
+        id-key  (reduce-kv (fn [_ k _] (when (= (json-ld/expand-iri k context) const/iri-id)
+                                         (reduced k))) nil orig)
+
+        p-sym-var  (::where/var p)
+        p-str-var  (str p-sym-var)
+        orig-p-iri (some-> (::where/iri p)
+                           (json-ld/compact context))]
+    ;; p-sym-var, p-str-var, orig-p-iri are mutually exclusive keys, only one will be present
+    ;; we cannot know whether the original var is a symbol or a str, so we just try both
+    (select-keys orig [id-key p-sym-var p-str-var orig-p-iri])))
+
+(defn display-pattern
+  "Replace parsed patterns with the user's original syntax."
+  [pattern]
+  (let [{:keys [orig context]} (meta pattern)]
+    (if (multi-triple-node-pattern? orig)
+      (display-triple-pattern pattern context orig)
+      orig)))
+
+(defn tally
+  "Format the explanation as a vector of [<pattern> <counters>] tuples in execution order."
+  [tracker]
+  (let [{:keys [patterns] :as explain} @tracker]
+    (reduce (fn [explanation pattern]
+              (conj explanation [(display-pattern pattern) (-> (get explain pattern)
+                                                               ;; display vars in order of insertion for readability
+                                                               (update :binds-in #(vec (sort-by (comp :ord meta) %)))
+                                                               (update :binds-out #(vec (sort-by (comp :ord meta) %))))]))
+            []
+            patterns)))