[GH-1624] Rename covid -> staged

api/deps.edn

@@ -96,14 +96,6 @@
    :extra-paths ["test"]
    :main-opts   ["-m" "wfl.tools.parallel-runner"]}
 
-  :demo
-  {:extra-deps
-   {org.apache.commons/commons-io {:mvn/version "1.3.2"}
-    org.liquibase/liquibase-core  {:mvn/version "4.3.5"
-                                   :exclusions  [ch.qos.logback/logback-classic]}}
-   :extra-paths ["test" "../derived/api/test/resources"]
-   :main-opts   ["-m" "wfl.system.cdc-covid19-surveillance-demo"]}
-
   :uberjar {:extra-deps {uberdeps/uberdeps {:mvn/version "1.0.4"}}}
 
   :prebuild {:extra-paths ["./build"] :exec-fn build/prebuild}

api/src/wfl/api/handlers.clj

@@ -1,5 +1,5 @@
 (ns wfl.api.handlers
-  "Define handlers for API endpoints. Require wfl.module namespaces here."
+  "Define handlers for API endpoints."
   (:require [clojure.set                    :refer [rename-keys]]
             [wfl.wfl                        :as wfl]
             [ring.util.http-response        :as response]
@@ -8,11 +8,6 @@
             [wfl.jdbc                       :as jdbc]
             [wfl.log                        :as log]
             [wfl.module.aou                 :as aou]
-            [wfl.module.copyfile]
-            [wfl.module.covid]
-            [wfl.module.sg]
-            [wfl.module.wgs]
-            [wfl.module.xx]
             [wfl.service.google.storage     :as gcs]
             [wfl.service.postgres           :as postgres]
             [wfl.util                       :as util])

api/src/wfl/api/spec.clj

@@ -7,8 +7,8 @@
             [wfl.module.aou       :as aou]
             [wfl.module.batch     :as batch]
             [wfl.module.copyfile  :as copyfile]
-            [wfl.module.covid     :as covid]
             [wfl.module.sg        :as sg]
+            [wfl.module.staged    :as staged]
             [wfl.module.wgs       :as wgs]
             [wfl.module.xx        :as xx]
             [wfl.util             :as util]))
@@ -46,10 +46,10 @@
 
 (s/def ::workflows (s/* ::workflow))
 
-(s/def ::workload-request (s/or :batch ::batch/workload-request
-                                :covid ::covid/workload-request))
+(s/def ::workload-request (s/or :batch  ::batch/workload-request
+                                :staged ::staged/workload-request))
 
-(s/def ::workload-response (s/or :batch ::batch/workload-response
-                                 :covid ::covid/workload-response))
+(s/def ::workload-response (s/or :batch  ::batch/workload-response
+                                 :staged ::staged/workload-response))
 
 (s/def ::workload-responses (s/* ::workload-response))

api/src/wfl/module/batch.clj

@@ -23,10 +23,11 @@
 (s/def ::creator string?)
 
 ;; This is the wrong thing to do. See [1] for more information.
-;; As a consequence, I've included the keys for a covid pipeline as optional
-;; inputs for batch workloads so that these keys are not removed during
-;; coercion.
+;; As a consequence, I've included the keys for a staged pipeline
+;; as optional inputs for batch workloads
+;; so that these keys are not removed during coercion.
 ;; [1]: https://github.com/metosin/reitit/issues/494
+;;
 (s/def ::workload-request
   (s/keys :opt-un [::all/common
                    ::all/input

api/src/wfl/module/covid.clj → api/src/wfl/module/staged.clj

@@ -1,5 +1,5 @@
-(ns wfl.module.covid
-  "Manage the Sarscov2IlluminaFull pipeline."
+(ns wfl.module.staged
+  "Manage staged (source -> executor -> sink) workloads."
   (:require [clojure.spec.alpha   :as s]
             [clojure.edn          :as edn]
             [wfl.api.workloads    :as workloads :refer [defoverload]]
@@ -92,7 +92,7 @@
    ,      sink_items     = ?
    WHERE  id = ?")
 
-(defn ^:private create-covid-workload
+(defn ^:private create-staged-workload
   [tx {:keys [source executor sink] :as request}]
   (let [id (add-workload-metadata tx request)]
     (jdbc/execute!
@@ -104,7 +104,7 @@
              [id]))
     (workloads/load-workload-for-id tx id)))
 
-(defn ^:private load-covid-workload-impl [tx {:keys [id] :as workload}]
+(defn ^:private load-staged-workload-impl [tx {:keys [id] :as workload}]
   (let [src-exc-sink {:source   (source/load-source! tx workload)
                       :executor (executor/load-executor! tx workload)
                       :sink     (sink/load-sink! tx workload)}]
@@ -115,7 +115,7 @@
       (filter second $)
       (into {:type :workload :id id} $))))
 
-(defn ^:private start-covid-workload
+(defn ^:private start-staged-workload
   "Start creating and managing workflows from the source."
   [tx {:keys [started] :as workload}]
   (letfn [(start [{:keys [id source] :as workload} now]
@@ -124,7 +124,7 @@
             (workloads/load-workload-for-id tx id))]
     (if-not started (start workload (utc-now)) workload)))
 
-(defn ^:private update-covid-workload
+(defn ^:private update-staged-workload
   "Use transaction `tx` to update `workload` statuses."
   [tx {:keys [started finished] :as workload}]
   (letfn [(update! [{:keys [id source executor sink] :as workload} now]
@@ -138,7 +138,7 @@
             (workloads/load-workload-for-id tx id))]
     (if (and started (not finished)) (update! workload (utc-now)) workload)))
 
-(defn ^:private stop-covid-workload
+(defn ^:private stop-staged-workload
   "Use transaction `tx` to stop the `workload` looking for new data."
   [tx {:keys [started stopped finished] :as workload}]
   (letfn [(stop! [{:keys [id source] :as workload} now]
@@ -152,7 +152,7 @@
                              {:workload workload})))
     (if-not (or stopped finished) (stop! workload (utc-now)) workload)))
 
-(defn ^:private retry-covid-workload
+(defn ^:private retry-staged-workload
   "Retry/resubmit the `workflows` managed by the `workload` and return the
    workload that manages the new workflows."
   [{:keys [started id executor] :as workload} workflows]
@@ -175,11 +175,11 @@
       (update :executor util/to-edn)
       (update :sink     util/to-edn)))
 
-(defoverload workloads/create-workload!     pipeline create-covid-workload)
-(defoverload workloads/start-workload!      pipeline start-covid-workload)
-(defoverload workloads/update-workload!     pipeline update-covid-workload)
-(defoverload workloads/stop-workload!       pipeline stop-covid-workload)
-(defoverload workloads/load-workload-impl   pipeline load-covid-workload-impl)
+(defoverload workloads/create-workload!     pipeline create-staged-workload)
+(defoverload workloads/start-workload!      pipeline start-staged-workload)
+(defoverload workloads/update-workload!     pipeline update-staged-workload)
+(defoverload workloads/stop-workload!       pipeline stop-staged-workload)
+(defoverload workloads/load-workload-impl   pipeline load-staged-workload-impl)
 (defmethod   workloads/workflows            pipeline
   [tx {:keys [executor] :as _workload}]
   (executor/executor-workflows tx executor {}))
@@ -188,5 +188,5 @@
   (executor/executor-workflows tx executor filters))
 (defoverload workloads/throw-if-invalid-retry-filters
   pipeline executor/executor-throw-if-invalid-retry-filters)
-(defoverload workloads/retry               pipeline retry-covid-workload)
+(defoverload workloads/retry               pipeline retry-staged-workload)
 (defoverload workloads/to-edn              pipeline workload-to-edn)

api/test/wfl/integration/modules/covid_test.clj → api/test/wfl/integration/modules/staged_test.clj

@@ -1,5 +1,4 @@
-(ns wfl.integration.modules.covid-test
-  "Test the Sarscov2IlluminaFull COVID pipeline."
+(ns wfl.integration.modules.staged-test
   (:require [clojure.test                   :refer [deftest is testing
                                                     use-fixtures]]
             [clojure.set                    :as set]
@@ -64,7 +63,7 @@
             (is (str/starts-with? details "TerraWorkspaceSink_")))]
     (let [{:keys [created creator source executor sink labels watchers] :as workload}
           (workloads/create-workload!
-           (workloads/covid-workload-request
+           (workloads/staged-workload-request
             {:skipValidation true}
             {:skipValidation true}
             {:skipValidation true}))]
@@ -79,7 +78,7 @@
       (is (vector? watchers)))))
 
 (deftest test-workload-to-edn
-  (let [request  (workloads/covid-workload-request
+  (let [request  (workloads/staged-workload-request
                   {:skipValidation true}
                   {:skipValidation true}
                   {:skipValidation true})]
@@ -99,36 +98,36 @@
       (is (not-any? (:executor workload) [:id :details :type]))
       (is (not-any? (:sink workload) [:id :details :type])))))
 
-(deftest test-create-covid-workload-with-misnamed-source
+(deftest test-create-staged-workload-with-misnamed-source
   (is (thrown-with-msg?
        UserException #"Invalid source"
        (workloads/create-workload!
-        (workloads/covid-workload-request
+        (workloads/staged-workload-request
          {:name "bad name"}
          {:skipValidation true}
          {:skipValidation true})))))
 
-(deftest test-create-covid-workload-with-misnamed-executor
+(deftest test-create-staged-workload-with-misnamed-executor
   (is (thrown-with-msg?
        UserException #"Invalid executor"
        (workloads/create-workload!
-        (workloads/covid-workload-request
+        (workloads/staged-workload-request
          {:skipValidation true}
          {:name "bad name"}
          {:skipValidation true})))))
 
-(deftest test-create-covid-workload-with-misnamed-sink
+(deftest test-create-staged-workload-with-misnamed-sink
   (is (thrown-with-msg?
        UserException #"Invalid sink"
        (workloads/create-workload!
-        (workloads/covid-workload-request
+        (workloads/staged-workload-request
          {:skipValidation true}
          {:skipValidation true}
          {:name "bad name"})))))
 
-(deftest test-create-covid-workload-with-valid-executor-request
+(deftest test-create-staged-workload-with-valid-executor-request
   (is (workloads/create-workload!
-       (workloads/covid-workload-request
+       (workloads/staged-workload-request
         {:skipValidation true}
         {:workspace                  testing-workspace
          :methodConfiguration        testing-method-configuration
@@ -138,7 +137,7 @@
 
 (deftest test-start-workload
   (let [workload (workloads/create-workload!
-                  (workloads/covid-workload-request
+                  (workloads/staged-workload-request
                    {:skipValidation true}
                    {:skipValidation true}
                    {:skipValidation true}))]
@@ -213,7 +212,7 @@
 
 (deftest test-get-workflows-empty
   (let [workload (workloads/create-workload!
-                  (workloads/covid-workload-request
+                  (workloads/staged-workload-request
                    {:skipValidation true}
                    {:skipValidation true}
                    {:skipValidation true}))]
@@ -231,22 +230,22 @@
      #'firecloud/get-submission               mock-firecloud-get-submission
      #'firecloud/get-workflow                 mock-workflow-keep-status}
     #(shared/run-workload-state-transition-test!
-      (workloads/covid-workload-request
+      (workloads/staged-workload-request
        {:skipValidation true}
        {:skipValidation true}
        {:skipValidation true}))))
 
-(deftest test-batch-workload-state-transition
+(deftest test-staged-workload-state-transition
   (shared/run-workload-state-transition-test!
-   (workloads/covid-workload-request
+   (workloads/staged-workload-request
     {:name      "TDR Snapshots"
      :snapshots []}
     {:skipValidation true}
     {:skipValidation true})))
 
 (deftest test-stop-workload-state-transition
   (shared/run-stop-workload-state-transition-test!
-   (workloads/covid-workload-request
+   (workloads/staged-workload-request
     {:skipValidation true}
     {:skipValidation true}
     {:skipValidation true})))
@@ -262,7 +261,7 @@
      #'firecloud/submit-method                mock-firecloud-create-submission
      #'firecloud/get-workflow                 (constantly {:status "Failed"})}
     #(shared/run-workload-state-transition-test!
-      (workloads/covid-workload-request
+      (workloads/staged-workload-request
        {:skipValidation true}
        {:skipValidation true}
        {:skipValidation true}))))
@@ -272,7 +271,7 @@
                  ::spec/workload-request
                  ::spec/workload-response
                  (comp util/to-edn workloads/create-workload!))
-        request (workloads/covid-workload-request
+        request (workloads/staged-workload-request
                  {}
                  {:skipValidation true}
                  {:skipValidation true})]
@@ -299,7 +298,7 @@
      #'firecloud/update-method-configuration  mock-firecloud-update-method-configuration
      #'firecloud/submit-method                mock-firecloud-create-submission
      #'firecloud/get-workflow                 (constantly {:status "Failed"})}
-    #(let [workload-request (workloads/covid-workload-request
+    #(let [workload-request (workloads/staged-workload-request
                              {:skipValidation true}
                              {:skipValidation true}
                              {:skipValidation true})

api/test/wfl/integration/source_test.clj

@@ -262,12 +262,12 @@
     (is (= (:snapshots source) [(:id snapshot)]))
     (is (s/valid? ::source/snapshot-list-source source))))
 
-(deftest test-create-covid-workload-with-empty-snapshot-list
+(deftest test-create-staged-workload-with-empty-snapshot-list
   (is (source/tdr-snapshot-list-validate-request-or-throw
        {:name      "TDR Snapshots"
         :snapshots [testing-snapshot]})))
 
-(deftest test-create-covid-workload-with-invalid-snapshot
+(deftest test-create-staged-workload-with-invalid-snapshot
   (is (thrown-with-msg?
        UserException #"Cannot access snapshot"
        (source/tdr-snapshot-list-validate-request-or-throw