refactors component config:

- supports mulitiple evaluation links - explicit specification of pre-flight evaluations - removes component/content map in favor of namespaced properties respective to component flavor and behavior
MichaelDrogalis · Oct 31, 2016 · 0ab90d1 · 0ab90d1
1 parent cf8063b
commit 0ab90d1
Show file tree

Hide file tree

Showing 8 changed files with 107 additions and 18,396 deletions.
diff --git a/resources/public/js/compiled/onyx_blueprint.js b/resources/public/js/compiled/onyx_blueprint.js
diff --git a/src/onyx_blueprint/api.cljs b/src/onyx_blueprint/api.cljs
@@ -14,8 +14,8 @@
      (cb {:component/id component-id
           :result result
           :state (if (seq (:warnings result))
-                   :compiled-error
-                   :compiled-success)}))))
+                   :error
+                   :success)}))))
 
 (defn io-evaluate [cb {:keys [component-id] :as eval-req}]
   (evaluate* eval-req (fn [result]
@@ -55,29 +55,27 @@
 
 (def run-async! #'cljs.js/run-async!)
 
-(defn eval-default-input! [components done-cb]
-  (let [evaluations (->> components
-                          (filter #(= :editor (keyword (namespace (:component/type %)))))
-                          (map #(vector (:component/id %)
-                                        (get-in % [:component/content :default-input])))
-                          (filter second))
+(defn initial-evaluations! [components done-cb]
+  (let [inits (keep (fn [{:keys [component/id evaluations/init] :as c}]
+                      (when init [id (get c init)]))
+                    components)
         results (atom {})]
     (run-async!
-     (fn [[id default-input] cb]
+     (fn [[id init-input] cb]
        (evaluate* {:component-id id
-                  :script-id (str (name id) "-initial-value")
-                  :source default-input}
-                 (fn [result]
-                   (swap! results assoc-in [id] result)
-                   (cb nil))))
-     evaluations
+                   :script-id (str (name id) "-initial-value")
+                   :source init-input}
+                  (fn [result]
+                    (swap! results assoc-in [id] result)
+                    (cb nil))))
+     inits
      :unused
      (fn [error-result]
        (done-cb @results)))))
 
 (defn render-tutorial! [components sections target-el]
   ;; todo alternative to blocking here?
-  (eval-default-input!
+  (initial-evaluations!
    components
    (fn [evaluations]
      (let [init-data {:blueprint/sections (into-tree components sections)}

diff --git a/src/onyx_blueprint/core.cljs b/src/onyx_blueprint/core.cljs
@@ -8,90 +8,92 @@
   [;; Workflow
    {:component/id ::workflow-header
     :component/type :text/header
-    :component/content {:text "Workflow"}}
+    :content/text "Workflow"}
 
    {:component/id ::workflow-leadin
     :component/type :text/body
-    :component/content
-    {:text ["A workflow is the structural specification of an Onyx program. Its purpose is to articulate the paths that data flows through the cluster at runtime. It is specified via a directed, acyclic graph."
-            "The workflow representation is a Clojure vector of vectors. Each inner vector contains exactly two elements, which are keywords. The keywords represent nodes in the graph, and the vector represents a directed edge from the first node to the second."
-            "[Try editing the example data structures ...]"]}}
+    :content/text
+    ["A workflow is the structural specification of an Onyx program. Its purpose is to articulate the paths that data flows through the cluster at runtime. It is specified via a directed, acyclic graph."
+     "The workflow representation is a Clojure vector of vectors. Each inner vector contains exactly two elements, which are keywords. The keywords represent nodes in the graph, and the vector represents a directed edge from the first node to the second."
+     "[Try editing the example data structures ...]"]}
 
    {:component/id ::workflow-ex1-data
     :component/type :editor/data-structure
-    :component/content {:default-input [[:input :increment] [:increment :output]]}}
+    :evaluations/init :content/default-input
+    :content/default-input [[:input :increment] [:increment :output]]}
 
    {:component/id ::workflow-ex1-graph
-    :component/type :graph/simple
-    :component/target ::workflow-ex1-data}
+    :component/type :graph/workflow
+    :evaluations/link {:workflow ::workflow-ex1-data}}
 
    {:component/id ::workflow-ex2-data
     :component/type :editor/data-structure
-    :component/content {:default-input [[:input :processing-1]
-                                        [:input :processing-2]
-                                        [:processing-1 :output-1]
-                                        [:processing-2 :output-2]]}}
+    :evaluations/init :content/default-input
+    :content/default-input [[:input :processing-1]
+                              [:input :processing-2]
+                              [:processing-1 :output-1]
+                              [:processing-2 :output-2]]}
 
    {:component/id ::workflow-ex2-graph
-    :component/type :graph/simple
-    :component/target ::workflow-ex2-data}
+    :component/type :graph/workflow
+    :evaluations/link {:workflow ::workflow-ex2-data}}
 
 
    {:component/id ::workflow-ex3-data
     :component/type :editor/data-structure
-    :component/content {:default-input [[:input :processing-1]
-                                        [:input :processing-2]
-                                        [:processing-1 :output]
-                                        [:processing-2 :output]]}}
+    :evaluations/init :content/default-input
+    :content/default-input [[:input :processing-1]
+                            [:input :processing-2]
+                            [:processing-1 :output]
+                            [:processing-2 :output]]}
 
    {:component/id ::workflow-ex3-graph
-    :component/type :graph/simple
-    :component/target ::workflow-ex3-data}
+    :component/type :graph/workflow
+    :evaluations/link {:workflow ::workflow-ex3-data}}
 
    ;; Catalog
    {:component/id ::catalog-header
     :component/type :text/header
-    :component/content {:text "Catalog"}}
+    :content/text "Catalog"}
 
    {:component/id ::catalog-leadin
     :component/type :text/body
-    :component/content
-    {:text "All inputs, outputs, and functions in a workflow must be described via a catalog. A catalog is a vector of maps, strikingly similar to Datomic’s schema. Configuration and docstrings are described in the catalog."}}
+    :content/text "All inputs, outputs, and functions in a workflow must be described via a catalog. A catalog is a vector of maps, strikingly similar to Datomic’s schema. Configuration and docstrings are described in the catalog."}
 
    ;; Basic job
    {:component/id ::job-ex1-header
     :component/type :text/header
-    :component/content {:text "Your first job"}}
+    :content/text "Your first job"}
 
    {:component/id ::job-ex1-leadin
     :component/type :text/body
-    :component/content
-    {:text "[Try editing this basic setup ...]"}}
+    :content/text "[Try editing this basic setup ...]"}
 
    {:component/id ::job-ex1-data
     :component/type :editor/data-structure
-    :component/content
-    {:default-input {:workflow [[:in :inc] [:inc :out]]
-                     :catalog [{:onyx/name :in
-                                :onyx/type :input
-                                :onyx/batch-size 20}
-                               {:onyx/name :inc
-                                :onyx/type :function
-                                :onyx/fn :cljs.user/my-inc
-                                :onyx/batch-size 20}
-                               {:onyx/name :out
-                                :onyx/type :output
-                                :onyx/batch-size 20}]}}}
+    :evaluations/init :content/default-input
+    :content/default-input {:workflow [[:in :inc] [:inc :out]]
+                            :catalog [{:onyx/name :in
+                                       :onyx/type :input
+                                       :onyx/batch-size 20}
+                                      {:onyx/name :inc
+                                       :onyx/type :function
+                                       :onyx/fn :cljs.user/my-inc
+                                       :onyx/batch-size 20}
+                                      {:onyx/name :out
+                                       :onyx/type :output
+                                       :onyx/batch-size 20}]}}
 
    {:component/id ::job-ex1-fn
     :component/type :editor/fn
-    :component/content
-    {:default-input "(defn ^:export my-inc [segment]\n  (update-in segment [:n] inc))"}}
+    :content/default-input
+    "(defn ^:export my-inc [segment]\n  (update-in segment [:n] inc))"}
 
    {:component/id ::job-ex1-simulator
     :component/type :simulator/basic
-    :component/content {:gen-segment (fn [] {:n (rand-int 100)})}
-    :component/target ::job-ex1-data}])
+    :simulator/gen-segment (fn [] {:n (rand-int 100)})
+    :evaluations/link {:init-data ::job-ex1-data
+                       :job-env ::job-ex1-simulator}}])
 
 (def sections
   [{:section/id ::workflow

diff --git a/src/onyx_blueprint/parser.cljs b/src/onyx_blueprint/parser.cljs
@@ -17,6 +17,14 @@
   (let [target-id (get data key)]
     {:value (get-in @state [:blueprint/evaluations target-id])}))
 
+(defmethod parser-read :evaluations/link
+  [{:keys [state query parser data] :as env} key _]
+  (let [refmap (get data key)]
+    {:value (reduce-kv (fn [m k refid]
+                         (assoc m k (get-in @state [:blueprint/evaluations refid])))
+                       {}
+                       refmap)}))
+
 (defmethod parser-read :row/items
   [{:keys [state query parser data] :as env} key _]
   (let [st @state
@@ -52,32 +60,36 @@
 
 (defmethod parser-mutate 'onyx/init
   [{:keys [state] :as env} key {:keys [id job]}]
-  {:action (fn []
-             (swap! state assoc-in
-                    [:blueprint/components id :component/content :job]
-                    (onyx.api/init job)))})
+  (let [job-env (onyx.api/init job)]
+    {:action (fn []
+               (swap! state assoc-in
+                      [:blueprint/evaluations id]
+                      ;; todo common record with api/evaluate* result
+                      {:component/id id
+                       :result {:value job-env}
+                       :warnings []
+                       :state :success}))}))
 
 (defmethod parser-mutate 'onyx/new-segment
   [{:keys [state] :as env} key {:keys [id]}]
-  {:action (fn []
-             (swap! state update-in
-                    [:blueprint/components id :component/content]
-                    (fn [{:keys [gen-segment] :as content}]
-                      (update-in content [:job]
-                                 #(onyx.api/new-segment % :in (gen-segment))))))})
+  (let [gen-segment (get-in @state [:blueprint/components id :simulator/gen-segment])]
+    {:action (fn []
+               (swap! state update-in
+                      [:blueprint/evaluations id :result :value]
+                      #(onyx.api/new-segment % :in (gen-segment))))}))
 
 (defmethod parser-mutate 'onyx/tick
   [{:keys [state] :as env} key {:keys [id]}]
   {:action (fn []
              (swap! state update-in
-                    [:blueprint/components id :component/content :job]
+                    [:blueprint/evaluations id :result :value]
                     #(onyx.api/tick %)))})
 
 (defmethod parser-mutate 'onyx/drain
   [{:keys [state] :as env} key {:keys [id]}]
   {:action (fn []
              (swap! state update-in
-                    [:blueprint/components id :component/content :job]
+                    [:blueprint/evaluations id :result :value]
                     #(-> %
                          (onyx.api/drain)
                          (onyx.api/stop))))})
diff --git a/src/onyx_blueprint/ui/code_editor.cljs b/src/onyx_blueprint/ui/code_editor.cljs
@@ -44,7 +44,7 @@
 (defui CodeEditor
     static om/IQuery
     (query [this]
-      [:component/id :component/type :component/content])
+      [:component/id :component/type :content/default-input])
 
     Object
     (componentDidMount [this]
@@ -61,9 +61,9 @@
                                     :blueprint/sections])))))
 
     (render [this]
-      (let [{:keys [component/id component/content] :as props} (om/props this)]
+      (let [{:keys [component/id content/default-input]} (om/props this)]
         (dom/textarea #js {:id (textarea-id id)
-                           :defaultValue (format-default-input (:default-input content))}))))
+                           :defaultValue (format-default-input default-input)}))))
 
 (def code-editor (om/factory CodeEditor))
 

diff --git a/src/onyx_blueprint/ui/graph.cljs b/src/onyx_blueprint/ui/graph.cljs
@@ -14,8 +14,8 @@
             (apply dep/depend g (reverse edge)))
           (dep/graph) workflow))
 
-(defn props->cytoscape-opts [{:keys [component/id component/target]}]
-  (let [workflow (-> target :result :value)
+(defn props->cytoscape-opts [{:keys [component/id evaluations/link]}]
+  (let [workflow (-> link :workflow :result :value)
         nodes (->> workflow
                    (flatten)
                    (set)
@@ -52,7 +52,7 @@
 (defui Graph
   static om/IQuery
   (query [this]
-    [:component/id :component/type :component/target])
+    [:component/id :component/type :evaluations/link])
 
   Object
   (componentDidMount [this]
@@ -69,5 +69,5 @@
 
 (def graph (om/factory Graph))
 
-(defmethod extensions/component-ui :graph/simple [props]
+(defmethod extensions/component-ui :graph/workflow [props]
   (graph props))
diff --git a/src/onyx_blueprint/ui/simulator.cljs b/src/onyx_blueprint/ui/simulator.cljs
@@ -14,23 +14,23 @@
 (defui Simulator
   static om/IQuery
   (query [this]
-    [:component/id :component/type :component/content :component/target])
+    [:component/id :component/type :evaluations/link])
 
   Object
   (render [this]
-    (let [{:keys [component/id component/content component/target] :as props} (om/props this)
-          {:keys [job]} content
-          init-data (-> target :result :value)
-          env-summary (if job (onyx.api/env-summary job))
+    (let [{:keys [component/id evaluations/link] :as props} (om/props this)
+          init-data (-> link :init-data :result :value)
+          job-env (-> link :job-env :result :value)
+          env-summary (if job-env (onyx.api/env-summary job-env))
           transact! (partial om/transact! this)]
       (apply dom/div #js {:id (name id) :className "col component component-editor"}
                (cond-> []
-                 (nil? job)
+                 (nil? job-env)
                  (conj (button "Initialize job"
                                (fn [evt]
                                  (.preventDefault evt)
                                  (transact! `[(onyx/init {:id ~id :job ~init-data})]))))
-                 job
+                 job-env
                  (conj (button "Reinitialize job"
                                (fn [evt]
                                  (.preventDefault evt)

diff --git a/src/onyx_blueprint/ui/text.cljs b/src/onyx_blueprint/ui/text.cljs
@@ -6,13 +6,12 @@
 (defui Text
   static om/IQuery
   (query [this]
-    [:component/id :component/type :component/content])
+    [:component/id :component/type :content/text])
 
   Object
   (render [this]
-    (let [{:keys [component/content component/type]} (om/props this)
-          {:keys [element]} (om/get-computed this)
-          text (:text content)]
+    (let [{:keys [component/content component/type content/text]} (om/props this)
+          {:keys [element]} (om/get-computed this)]
       (if (coll? text)
         (apply dom/div nil
                (map (partial element #js {:className (name type)})