Explain Analyze

src/fluree/db/api.cljc

@@ -698,14 +698,21 @@
     q - Query map (JSON-LD or analytical)
 
   Returns promise resolving to a query plan map with:
-    :query - Original parsed query structure
+    :query - Original query structure
     :plan - Execution plan with:
-      :optimization - :reordered, :unchanged, or :none
-      :statistics - Available statistics (if any)
+      :optimizations - array of optimization methods applied: :statistics, :heuristics, or :none
+      :statistics - If the :statistics optimization was applied, the statistics used as a basis
+         :properties - count of properties
+         :classes - count of classes
+         :flakes - total number of flakes in db
+         :index-t - t value of latest index
+         :segments - groups of patterns that are either optimizable or not
+      :heuristics - If the :heuristics optimization was applied, the list of optimized pattern types
       :original - Original pattern order with selectivity
       :optimized - Optimized pattern order with selectivity
-      :segments - Pattern segments with boundaries
-      :changed? - Boolean indicating if patterns were reordered
+
+  Patterns with higher selectivity scores are processed first.
+
 
   Example:
     @(fluree/explain db
@@ -718,17 +725,18 @@
 
     ;; Returns:
     {:query {...}
-     :plan {:optimization :reordered
-            :statistics {...}
+     :plan {:optimizations [:statistics]
             :original [{:pattern ... :selectivity 10000}
                        {:pattern ... :selectivity 1}]
             :optimized [{:pattern ... :selectivity 1}    ; email lookup first
                         {:pattern ... :selectivity 10000}] ; then verify type
-            :changed? true}}"
-  [ds q]
-  (if (util/exception? ds)
-    (throw ds)
-    (promise-wrap (query-api/explain ds q))))
+            :statistics {...}}}"
+  ([ds q]
+   (explain ds q {}))
+  ([ds q opts]
+   (if (util/exception? ds)
+     (throw ds)
+     (promise-wrap (query-api/explain ds q opts)))))
 
 (defn credential-query
   "Executes a query using a verifiable credential.

src/fluree/db/async_db.cljc

@@ -206,15 +206,20 @@
       root-db))
 
   optimize/Optimizable
-  (-reorder [_ parsed-query]
+  (-plan [_ parsed-query]
     (go-try
       (let [db (<? db-chan)]
-        (<? (optimize/-reorder db parsed-query)))))
+        (<? (optimize/-plan db parsed-query)))))
 
-  (-explain [_ parsed-query]
+  (-reorder [_ planned-query]
     (go-try
       (let [db (<? db-chan)]
-        (<? (optimize/-explain db parsed-query))))))
+        (<? (optimize/-reorder db planned-query)))))
+
+  (-explain [_ planned-query]
+    (go-try
+      (let [db (<? db-chan)]
+        (<? (optimize/-explain db planned-query))))))
 
 (defn db?
   [x]

src/fluree/db/dataset.cljc

@@ -130,16 +130,18 @@
                  nil)))))
 
   optimize/Optimizable
-  (-reorder [_ds parsed-query]
+  (-plan [_ds parsed-query]
+    (async/go parsed-query))
+  (-reorder [_ds planned-query]
     ;; DataSets (federated queries) are not optimized
     ;; Return query unchanged wrapped in channel
-    (async/go parsed-query))
+    (async/go planned-query))
 
-  (-explain [_ds parsed-query]
+  (-explain [_ds planned-query]
     ;; DataSets (federated queries) cannot be explained
     ;; Return simple plan indicating no optimization
     (async/go
-      {:query parsed-query
+      {:query planned-query
        :plan  {:optimization :none
                :reason       "Federated queries are not optimized"}})))
 

src/fluree/db/flake/flake_db.cljc

@@ -27,6 +27,7 @@
             [fluree.db.json-ld.vocab :as vocab]
             [fluree.db.query.exec.select.subject :as subject]
             [fluree.db.query.exec.where :as where]
+            [fluree.db.query.explain :as explain]
             [fluree.db.query.fql :as fql]
             [fluree.db.query.history :refer [AuditLog]]
             [fluree.db.query.optimize :as optimize]
@@ -40,8 +41,7 @@
             [fluree.db.util.log :as log]
             [fluree.db.util.reasoner :as reasoner-util]
             [fluree.db.virtual-graph.flat-rank :as flat-rank]
-            [fluree.db.virtual-graph.index-graph :as vg]
-            [fluree.json-ld :as json-ld])
+            [fluree.db.virtual-graph.index-graph :as vg])
   #?(:clj (:import (java.io Writer))))
 
 #?(:clj (set! *warn-on-reflection* true))
@@ -294,128 +294,6 @@
           (merge-flakes t-new all-flakes)
           (assoc :commit commit-metadata)))))
 
-(defn- component->user-value
-  "Convert an internal pattern component to user-readable format"
-  [component compact-fn]
-  (cond
-    (nil? component)
-    nil
-
-    (where/unmatched-var? component)
-    (str (where/get-variable component))
-
-    (where/matched-iri? component)
-    (let [iri (where/get-iri component)]
-      (json-ld/compact iri compact-fn))
-
-    (where/matched-value? component)
-    (where/get-value component)
-
-    :else
-    (throw (ex-info (str "Unexpected component type: " (pr-str component))
-                    {:component component}))))
-
-(defn- pattern->user-format
-  "Convert internal pattern to user-readable triple format"
-  [pattern compact-fn]
-  (let [ptype (where/pattern-type pattern)
-        pdata (where/pattern-data pattern)]
-    (case ptype
-      :class
-      (let [[s _ o] pdata]
-        {:subject (component->user-value s compact-fn)
-         :property const/iri-type
-         :object (component->user-value o compact-fn)})
-
-      :tuple
-      (let [[s p o] pdata]
-        {:subject (component->user-value s compact-fn)
-         :property (component->user-value p compact-fn)
-         :object (component->user-value o compact-fn)})
-
-      :id
-      {:subject (component->user-value pdata compact-fn)}
-
-      ;; Other pattern types (filter, bind, etc.)
-      {:type ptype
-       :data (pr-str pdata)})))
-
-(defn- pattern-type->user-type
-  "Convert internal pattern type to user-friendly type name"
-  [ptype]
-  (case ptype
-    :tuple :triple
-    ptype))
-
-(defn- pattern-explain
-  "Generate explain information for a single pattern"
-  [db stats pattern compact-fn]
-  (let [ptype       (where/pattern-type pattern)
-        optimizable? (optimize/optimizable-pattern? pattern)
-        selectivity (optimize/calculate-selectivity db stats pattern)]
-    {:type        (pattern-type->user-type ptype)
-     :pattern     (pattern->user-format pattern compact-fn)
-     :selectivity selectivity
-     :optimizable (when optimizable? (pattern-type->user-type optimizable?))}))
-
-(defn- segment-explain
-  "Generate explain information for pattern segments"
-  [db stats where-clause compact-fn]
-  (let [segments (optimize/split-by-optimization-boundaries where-clause)]
-    (mapv (fn [segment]
-            (if (= :optimizable (:type segment))
-              {:type     :optimizable
-               :patterns (mapv #(pattern-explain db stats % compact-fn) (:data segment))}
-              {:type    :boundary
-               :pattern (pattern-explain db stats (:data segment) compact-fn)}))
-          segments)))
-
-(defn optimize-query
-  "Optimize a parsed query using statistics if available.
-   Returns the optimized query with patterns reordered for optimal execution."
-  [db parsed-query]
-  (let [stats (:stats db)]
-    (if (and stats (not-empty stats) (:where parsed-query))
-      (let [optimized-where (optimize/optimize-patterns db (:where parsed-query))]
-        (assoc parsed-query :where optimized-where))
-      parsed-query)))
-
-(defn explain-query
-  "Generate an execution plan for the query showing optimization details.
-   Returns a query plan map with optimization information."
-  [db parsed-query]
-  (let [stats            (:stats db)
-        has-stat-counts? (and stats
-                              (or (seq (:properties stats))
-                                  (seq (:classes stats))))
-        context          (:context parsed-query)
-        compact-fn       (json-ld/compact-fn context)]
-    (if-not has-stat-counts?
-      {:query parsed-query
-       :plan  {:optimization :none
-               :reason       "No statistics available"
-               :where-clause (:where parsed-query)}}
-      (let [where-clause      (:where parsed-query)
-            optimized-where   (when where-clause
-                                (optimize/optimize-patterns db where-clause))
-            original-explain  (when where-clause
-                                (mapv #(pattern-explain db stats % compact-fn) where-clause))
-            optimized-explain (when optimized-where
-                                (mapv #(pattern-explain db stats % compact-fn) optimized-where))
-            segments          (when where-clause
-                                (segment-explain db stats where-clause compact-fn))
-            changed?          (not= where-clause optimized-where)]
-        {:query (assoc parsed-query :where optimized-where)
-         :plan  {:optimization (if changed? :reordered :unchanged)
-                 :statistics   {:property-counts (count (:properties stats))
-                                :class-counts    (count (:classes stats))
-                                :total-flakes    (:flakes stats)
-                                :indexed-at-t    (:indexed stats)}
-                 :original     original-explain
-                 :optimized    optimized-explain
-                 :segments     segments
-                 :changed?     changed?}}))))
-
 (defrecord FlakeDB [index-catalog commit-catalog alias commit t tt-id stats
                     spot post opst tspo vg schema comparators staged novelty policy
                     namespaces namespace-codes max-namespace-code
@@ -599,11 +477,12 @@
     (reasoner-util/reasoned-facts db))
 
   optimize/Optimizable
-  (-reorder [db parsed-query]
-    (async/go (optimize-query db parsed-query)))
-
-  (-explain [db parsed-query]
-    (async/go (explain-query db parsed-query))))
+  (-plan [db parsed-query]
+    (async/go (optimize/plan-query db parsed-query)))
+  (-reorder [_db planned-query]
+    (async/go (optimize/optimize-query planned-query)))
+  (-explain [db planned-query]
+    (async/go (explain/query (:stats db) planned-query))))
 
 (defn db?
   [x]

src/fluree/db/query/api.cljc

@@ -5,9 +5,12 @@
             [fluree.db.dataset :as dataset :refer [dataset?]]
             [fluree.db.json-ld.policy :as perm]
             [fluree.db.ledger :as ledger]
+            [fluree.db.query.exec :as exec]
             [fluree.db.query.fql :as fql]
+            [fluree.db.query.fql.parse :as parse]
             [fluree.db.query.fql.syntax :as syntax]
             [fluree.db.query.history :as history]
+            [fluree.db.query.optimize :as optimize]
             [fluree.db.query.sparql :as sparql]
             [fluree.db.reasoner :as reasoner]
             [fluree.db.time-travel :as time-travel]
@@ -144,17 +147,32 @@
 
   Parameters:
     db - Database value or dataset
-    query - Query map (JSON-LD or analytical)
+    q - Query map (JSON-LD or analytical)
 
   Returns channel resolving to a query plan map."
-  [db query]
+  [db q override-opts]
   (go-try
-    (let [q (-> query
-                syntax/coerce-query
-                (sanitize-query-options nil))
-          q* (update q :opts dissoc :meta :max-fuel)]
+    (let [{:keys [opts] :as parsed-query}
+          (-> q
+              syntax/coerce-query
+              (sanitize-query-options override-opts)
+              (assoc :orig-query q)
+              (assoc-in [:opts :meta] true)
+              parse/parse-query*)
 
-      (<? (fql/explain db q*)))))
+          planned-query (<? (optimize/-plan db parsed-query))]
+      (if (track/track-solutions? opts)
+        (let [optimized-query (<? (optimize/-reorder db planned-query))
+              tracker         (track/init opts)
+
+              db* (if (dataset? db)
+                    db
+                    (<? (restrict-db db tracker optimized-query)))
+
+              result      (<? (track-execution db* tracker #(exec/query db* tracker optimized-query)))
+              explanation (<? (optimize/-explain db* planned-query))]
+          (merge result explanation))
+        (<? (optimize/-explain db planned-query))))))
 
 (defn contextualize-ledger-400-error
   [info-str e]

src/fluree/db/query/exec/where.cljc

@@ -318,7 +318,7 @@
   [pattern]
   (if (map-entry? pattern)
     (key pattern)
-    :tuple))
+    :triple))
 
 (defn pattern-data
   [pattern]
@@ -333,6 +333,16 @@
   (fn [_ds _tracker _solution pattern _error-ch]
     (pattern-type pattern)))
 
+(defn match-and-track-pattern
+  [ds tracker solution pattern error-ch]
+  (if (:analyze tracker)
+    (do (track/pattern-in! tracker pattern solution)
+        (-> (match-pattern ds tracker solution pattern error-ch)
+            (async/pipe (async/chan 2 (map (fn [solution]
+                                             (track/pattern-out! tracker pattern solution)
+                                             solution))))))
+    (match-pattern ds tracker solution pattern error-ch)))
+
 (defn assign-solution-filter
   [component solution]
   (if (::fn component)
@@ -585,10 +595,10 @@
   (let [s-mch (pattern-data pattern)]
     (-match-id ds tracker solution s-mch error-ch)))
 
-(defmethod match-pattern :tuple
+(defmethod match-pattern :triple
   [ds tracker solution pattern error-ch]
-  (let [tuple (pattern-data pattern)]
-    (-match-triple ds tracker solution tuple error-ch)))
+  (let [triple (pattern-data pattern)]
+    (-match-triple ds tracker solution triple error-ch)))
 
 (defmethod match-pattern :class
   [ds tracker solution pattern error-ch]
@@ -652,7 +662,7 @@
     (async/pipeline-async 2
                           out-ch
                           (fn [solution ch]
-                            (-> (match-pattern ds tracker solution pattern error-ch)
+                            (-> (match-and-track-pattern ds tracker solution pattern error-ch)
                                 (async/pipe ch)))
                           solution-ch)
     out-ch))