diff --git a/CHANGELOG.md b/CHANGELOG.md index 12d994706..53017df7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# Release 0.12.1 +- [GH-1586] Run AoU pipeline with the Arrays_v2.6.3 WDL. ([#557](https://github.com/broadinstitute/wfl/pull/557)) +- [GH-1527] Remove Firecloud calls when fetching TerraExecutor workflows ([#543](https://github.com/broadinstitute/wfl/pull/543)) + # Release 0.12.0 - [GH-1581] Run AoU pipeline with the Arrays_v2.6.2 WDL. ([#554](https://github.com/broadinstitute/wfl/pull/554)) - [GH-1540] Apply User Comments to Terra Submissions ([#551](https://github.com/broadinstitute/wfl/pull/551)) diff --git a/api/src/wfl/api/spec.clj b/api/src/wfl/api/spec.clj index 9847fb089..6ec22d1c2 100644 --- a/api/src/wfl/api/spec.clj +++ b/api/src/wfl/api/spec.clj @@ -41,8 +41,8 @@ :sg ::sg/workflow-inputs :other map?)) -(s/def ::workflow (s/keys :req-un [::inputs] - :opt-un [::all/status ::all/updated ::all/uuid ::all/options])) +(s/def ::workflow (s/or :batch ::batch/workflow + :staged ::executor/executor-workflow)) (s/def ::workflows (s/* ::workflow)) diff --git a/api/src/wfl/executor.clj b/api/src/wfl/executor.clj index 587c58271..6201589c5 100644 --- a/api/src/wfl/executor.clj +++ b/api/src/wfl/executor.clj @@ -32,12 +32,8 @@ (fn [{:keys [executor] :as _workload}] (:type executor))) (defmulti executor-workflows - "Return the workflows managed by the `executor" - (fn [_transaction executor] (:type executor))) - -(defmulti executor-workflows-by-filters "Use db `transaction` to return the workflows created by the `executor` - matching `filters` (ex. status, submission)." + matching optional `filters` (ex. status, submission)." (fn [_transaction executor _filters] (:type executor))) (defmulti executor-throw-if-invalid-retry-filters @@ -89,12 +85,27 @@ :fromSource :from_source}) ;; Specs +(s/def ::entity ::all/uuid) +(s/def ::fromSource string?) +(s/def ::methodConfiguration (s/and string? util/terra-namespaced-name?)) +(s/def ::methodConfigurationVersion integer?) +(s/def ::reference ::all/uuid) +(s/def ::submission ::all/uuid) +(s/def ::workflow ::all/uuid) + (s/def ::terra-executor (s/keys :req-un [::all/name - ::all/fromSource - ::all/methodConfiguration - ::all/methodConfigurationVersion + ::fromSource + ::methodConfiguration + ::methodConfigurationVersion ::all/workspace])) -(s/def ::submission util/uuid-string?) + +(s/def ::terra-executor-workflow (s/keys :req-un [::entity + ::methodConfiguration + ::reference + ::submission + ::all/workspace] + :opt-un [::workflow + ::all/status])) (defn ^:private write-terra-executor [tx id executor] (let [create "CREATE TABLE %s OF TerraExecutorDetails (PRIMARY KEY (id))" @@ -261,11 +272,12 @@ "Assign workflow uuids from previous submissions" [{:keys [workspace details] :as executor}] (letfn [(read-a-submission-without-workflows [] - (let [query "SELECT id, submission, entity FROM %s - WHERE submission IN ( - SELECT submission FROM %s WHERE workflow IS NULL - LIMIT 1 - )"] + (let [query (str/join \space ["SELECT id, submission, entity FROM %s" + "WHERE submission IN (" + "SELECT submission FROM %s" + "WHERE workflow IS NULL" + "LIMIT 1" + ")"])] (jdbc/with-db-transaction [tx (postgres/wfl-db-config)] (let [records (jdbc/query tx (format query details details))] (when-first [{:keys [submission]} records] @@ -315,7 +327,7 @@ [{:keys [workspace details] :as executor}] (letfn [(update-status-from-firecloud [{:keys [submission workflow] :as record}] - (->> (firecloud/get-workflow workspace submission workflow) + (->> (firecloud/get-workflow workspace submission workflow "status") :status (assoc record :status))) (write-workflow-statuses [now records] @@ -365,11 +377,11 @@ (defn ^:private peek-terra-executor-details "Get first unconsumed successful workflow record from `details` table." [{:keys [details] :as _executor}] - (let [query "SELECT * FROM %s - WHERE consumed IS NULL - AND status = 'Succeeded' - ORDER BY id ASC - LIMIT 1"] + (let [query (str/join \space ["SELECT * FROM %s" + "WHERE consumed IS NULL" + "AND status = 'Succeeded'" + "ORDER BY id ASC" + "LIMIT 1"])] (jdbc/with-db-transaction [tx (postgres/wfl-db-config)] (->> (format query details) (jdbc/query tx) @@ -421,28 +433,18 @@ "Return the number of workflows in the `executor` queue that are yet to be consumed." [{:keys [details] :as _executor}] - (let [query "SELECT COUNT(*) FROM %s - WHERE consumed IS NULL - AND (status IS NULL OR - status NOT IN ('Failed', 'Aborted'))"] + (let [query (str/join \space ["SELECT COUNT(*) FROM %s" + "WHERE consumed IS NULL" + "AND (status IS NULL OR" + "status NOT IN ('Failed', 'Aborted'))"])] (jdbc/with-db-transaction [tx (postgres/wfl-db-config)] (->> (format query details) (jdbc/query tx) first :count)))) -(defn ^:private terra-workflows-from-records - [{:keys [workspace] :as _executor} records] - (letfn [(from-record [{:keys [workflow submission status] :as record}] - (combine-record-workflow-and-outputs - record - (firecloud/get-workflow workspace submission workflow) - (when (= "Succeeded" status) - (firecloud/get-workflow-outputs workspace submission workflow))))] - (map from-record records))) - -;; terra-executor-workflows and terra-executor-workflows-by-filters do not return -;; workflows that are being or have been retried. Why? +;; terra-executor-workflows does not return workflows +;; that are being or have been retried. Why? ;; ;; TL/DR: It's simpler maybe? ;; @@ -482,19 +484,6 @@ ;; not already an active workflow at the `HEAD`. ;; ;; ** Admittedly, this is still a problem with this design. - -(defn ^:private terra-executor-workflows - "Return all the non-retried workflows executed by the `executor`." - [tx {:keys [details] :as executor}] - (postgres/throw-unless-table-exists tx details) - (let [query "SELECT * FROM %s - WHERE workflow IS NOT NULL - AND retry IS NULL - ORDER BY id ASC"] - (terra-workflows-from-records - executor - (jdbc/query tx (format query details))))) - (defn ^:private remove-nil-and-join "Remove nil elements of `vec` and join with `separator`." ([vec separator] @@ -502,54 +491,54 @@ ([vec] (remove-nil-and-join vec \space))) -(defn ^:private terra-executor-workflows-by-filters-sql-params +(defn ^:private terra-executor-workflows-sql-params "Return sql and params that query `details` for non-retried workflows matching `submission` and/or `status` if specified." [{:keys [details] :as _executor} {:keys [submission status] :as _filters}] - (let [optionals (remove-nil-and-join [(when submission "AND submission = ?") - (when status "AND status = ?")]) - query (remove-nil-and-join ["SELECT * FROM %s" - "WHERE workflow IS NOT NULL" - "AND retry IS NULL" - optionals - "ORDER BY id ASC"])] + (let [query (remove-nil-and-join ["SELECT * FROM %s" + "WHERE workflow IS NOT NULL" + "AND retry IS NULL" + (when submission "AND submission = ?") + (when status "AND status = ?") + "ORDER BY id ASC"])] (->> [submission status] (concat [(format query details)]) (remove nil?)))) -(defn ^:private terra-executor-workflows-by-filters +(defn ^:private terra-executor-workflows "Return all the non-retried workflows executed by `executor` matching specified `filters`." [tx {:keys [details] :as executor} filters] (postgres/throw-unless-table-exists tx details) - (->> (terra-executor-workflows-by-filters-sql-params executor filters) - (jdbc/query tx) - (terra-workflows-from-records executor))) - + (let [sql-params (terra-executor-workflows-sql-params executor filters) + executor-subset (select-keys executor [:workspace :methodConfiguration])] + (map #(merge executor-subset %) (jdbc/query tx sql-params)))) + +;; 1. Presently Rawls doesn't support submitting a subset of a snapshot: +;; If we wish to retry 1+ workflows stemming from a snapshot, +;; we must resubmit the snapshot in its entirety. +;; 2. Though we are technically resubmitting snapshot references +;; and not submissions, this query takes a submission-based approach. +;; Why? Because there is no ambiguity when linking a submission +;; to its workflows. With the capacity for retries, one reference +;; may map to multiple sets of workflows, and retrying the reference +;; should only update the sibling workflow records of those supplied. +;; (defn ^:private workflow-and-sibling-records "Return the workflow records for all workflows in submissions associated with the specified `workflows` -- i.e. records for `workflows` and their siblings." [tx {:keys [details] :as executor} workflows] (postgres/throw-unless-table-exists tx details) - (let [workflow-ids (util/to-quoted-comma-separated-list (map :uuid workflows)) - _ (-> (str "%s For workflows %s, " - "fetching sibling workflow records by submission") - (format (log-prefix executor) workflow-ids) - log/debug) - ;; 1. Presently Rawls doesn't support submitting a subset of a snapshot: - ;; If we wish to retry 1+ workflows stemming from a snapshot, - ;; we must resubmit the snapshot in its entirety. - ;; 2. Though we are technically resubmitting snapshot references - ;; and not submissions, this query takes a submission-based approach. - ;; Why? Because there is no ambiguity when linking a submission - ;; to its workflows. With the capacity for retries, one reference - ;; may map to multiple sets of workflows, and retrying the reference - ;; should only update the sibling workflow records of those supplied. - query "SELECT * FROM %s - WHERE submission IN - (SELECT DISTINCT submission FROM %s - WHERE workflow IN %s)"] + (let [workflow-ids (util/to-quoted-comma-separated-list + (map :workflow workflows)) + query (str/join \space ["SELECT * FROM %s" + "WHERE submission IN" + "(SELECT DISTINCT submission FROM %s" + "WHERE workflow IN %s)"])] + (log/debug {:action "Fetching workflow-ids' sibling workflow records" + :executor executor + :workflow-ids workflow-ids}) (jdbc/query tx (format query details details workflow-ids)))) (defn update-retried-workflow-records @@ -632,14 +621,15 @@ :status 400} (into {} (map :data errors)))))))) +;; Further work required to deal in generic entities +;; rather than assumed snapshot references: +;; https://broadinstitute.atlassian.net/browse/GH-1422 +;; (defn ^:private terra-executor-retry-workflows "Resubmit the snapshot references associated with `workflows` in `workspace` and update each original workflow record with the row ID of its retry." [{{:keys [workspace] :as executor} :executor :as workload} workflows] (letfn [(submit-reference [reference-id] - ;; Further work required to deal in generic entities - ;; rather than assumed snapshot references: - ;; https://broadinstitute.atlassian.net/browse/GH-1422 (let [reference (rawls/get-snapshot-reference workspace reference-id) userComment (create-user-comment "Retry" workload (get-in reference [:attributes :snapshot]))] (->> reference @@ -666,7 +656,6 @@ (defoverload load-executor! terra-executor-type load-terra-executor) (defoverload update-executor! terra-executor-type update-terra-executor) (defoverload executor-workflows terra-executor-type terra-executor-workflows) -(defoverload executor-workflows-by-filters terra-executor-type terra-executor-workflows-by-filters) (defoverload executor-throw-if-invalid-retry-filters terra-executor-type terra-executor-throw-if-invalid-retry-filters) (defoverload executor-retry-workflows! terra-executor-type terra-executor-retry-workflows) @@ -678,8 +667,6 @@ (defoverload util/to-edn terra-executor-type terra-executor-to-edn) -;; reitit http coercion specs for an executor -;; Recall s/or doesn't work (https://github.com/metosin/reitit/issues/494) -(s/def ::executor - #(condp = (:name %) - terra-executor-name (s/valid? ::terra-executor %))) +;; Generic executor-level specs following all implementations +(s/def ::executor ::terra-executor) +(s/def ::executor-workflow ::terra-executor-workflow) diff --git a/api/src/wfl/jdbc.clj b/api/src/wfl/jdbc.clj index e70e936fb..d0bb9fb91 100644 --- a/api/src/wfl/jdbc.clj +++ b/api/src/wfl/jdbc.clj @@ -25,14 +25,18 @@ (let [{:keys [line]} (meta &form)] `(let [db# ~db sql-params# ~sql-params] - (logger/debug (str/join " " ["jdbc/query:" (format-db db#) sql-params#]) ~line) + (logger/debug + (str/join " " ["jdbc/query:" (format-db db#) (pr-str sql-params#)]) + ~line) (jdbc/query db# sql-params#)))) ([db sql-params opts] (let [{:keys [line]} (meta &form)] `(let [db# ~db sql-params# ~sql-params opts# ~opts] - (logger/debug (str/join " " ["jdbc/query:" (format-db db#) sql-params# opts#]) ~line) + (logger/debug + (str/join " " ["jdbc/query:" (format-db db#) (pr-str sql-params#) opts#]) + ~line) (jdbc/query db# sql-params# opts#))))) (defmacro update! @@ -88,14 +92,18 @@ (let [{:keys [line]} (meta &form)] `(let [db# ~db sql-params# ~sql-params] - (logger/info (str/join " " ["jdbc/execute!" (format-db db#) sql-params#]) ~line) + (logger/info + (str/join " " ["jdbc/execute!" (format-db db#) (pr-str sql-params#)]) + ~line) (jdbc/execute! db# sql-params#)))) ([db sql-params opts] (let [{:keys [line]} (meta &form)] `(let [db# ~db sql-params# ~sql-params opts# ~opts] - (logger/info (str/join " " ["jdbc/execute!" (format-db db#) sql-params# opts#]) ~line) + (logger/info + (str/join " " ["jdbc/execute!" (format-db db#) (pr-str sql-params#) opts#]) + ~line) (jdbc/execute! db# sql-params# opts#))))) (defmacro db-do-commands diff --git a/api/src/wfl/module/all.clj b/api/src/wfl/module/all.clj index 5a536c19b..5075b1201 100644 --- a/api/src/wfl/module/all.clj +++ b/api/src/wfl/module/all.clj @@ -135,11 +135,8 @@ (s/def ::common map?) (s/def ::entityType string?) -(s/def ::fromSource string?) (s/def ::labels (s/* util/label?)) (s/def ::name string?) -(s/def ::methodConfiguration (s/and string? util/terra-namespaced-name?)) -(s/def ::methodConfigurationVersion integer?) (s/def ::watcher (s/or :email slack/email-watcher? :slack slack/slack-channel-watcher?)) diff --git a/api/src/wfl/module/aou.clj b/api/src/wfl/module/aou.clj index 73f57bccf..646914b6d 100644 --- a/api/src/wfl/module/aou.clj +++ b/api/src/wfl/module/aou.clj @@ -33,7 +33,7 @@ (def workflow-wdl "The top-level WDL file and its version." - {:release "Arrays_v2.6.2" + {:release "Arrays_v2.6.3" :path "pipelines/broad/arrays/single_sample/Arrays.wdl"}) (def cromwell-label-map diff --git a/api/src/wfl/module/batch.clj b/api/src/wfl/module/batch.clj index e338892f4..e4fc0a5d9 100644 --- a/api/src/wfl/module/batch.clj +++ b/api/src/wfl/module/batch.clj @@ -56,6 +56,12 @@ ::all/uuid ::all/version])) +(s/def ::workflow (s/keys :opt-un [::all/options + ::all/status + ::all/updated + ::all/uuid] + :req-un [:wfl.api.spec/inputs])) + (defn ^:private cromwell-status "`status` of the workflow with `uuid` in `cromwell`." [cromwell uuid] diff --git a/api/src/wfl/module/covid.clj b/api/src/wfl/module/covid.clj index 395adff5b..4be515cc6 100644 --- a/api/src/wfl/module/covid.clj +++ b/api/src/wfl/module/covid.clj @@ -182,10 +182,10 @@ (defoverload workloads/load-workload-impl pipeline load-covid-workload-impl) (defmethod workloads/workflows pipeline [tx {:keys [executor] :as _workload}] - (executor/executor-workflows tx executor)) + (executor/executor-workflows tx executor {})) (defmethod workloads/workflows-by-filters pipeline [tx {:keys [executor] :as _workload} filters] - (executor/executor-workflows-by-filters tx executor filters)) + (executor/executor-workflows tx executor filters)) (defoverload workloads/throw-if-invalid-retry-filters pipeline executor/executor-throw-if-invalid-retry-filters) (defoverload workloads/retry pipeline retry-covid-workload) diff --git a/api/src/wfl/service/firecloud.clj b/api/src/wfl/service/firecloud.clj index b29c6e6f7..8181f97ab 100644 --- a/api/src/wfl/service/firecloud.clj +++ b/api/src/wfl/service/firecloud.clj @@ -137,11 +137,19 @@ [workspace submission-id] (get-workspace-json workspace "submissions" submission-id)) +;; Further work needed if we needed to specify multiple `include-key`s. +;; (defn get-workflow "Query the `firecloud-url` for the the `workflow` created by the `submission` - in the Terra `workspace`." - [workspace submission-id workflow-id] - (get-workspace-json workspace "submissions" submission-id "workflows" workflow-id)) + in the Terra `workspace`, optionally restricting output to `include-key`." + ([workspace submission-id workflow-id include-key] + (let [parts [workspace "submissions" submission-id "workflows" workflow-id]] + (-> (apply workspace-api-url parts) + (http/get {:headers (auth/get-auth-header) + :query-params (when include-key {:includeKey include-key})}) + (util/response-body-json)))) + ([workspace submission-id workflow-id] + (get-workflow workspace submission-id workflow-id nil))) (defn get-workflow-outputs "Query the `firecloud-url` for the outputs of the `workflow` created by @@ -149,16 +157,6 @@ [workspace submission-id workflow-id] (get-workspace-json workspace "submissions" submission-id "workflows" workflow-id "outputs")) -(defn get-workflow-status-by-entity - "Get workflow status given a Terra submission-id and entity-name." - [workspace {:keys [uuid inputs] :as _item}] - (let [name (:entity-name inputs)] - (->> (get-submission workspace uuid) - :workflows - (filter #(= name (get-in % [:workflowEntity :entityName]))) - first - :status))) - (defn get-entity "Fetch the `entity` metadata from the `workspace`." [workspace [type name :as _entity]] diff --git a/api/test/wfl/integration/executor_test.clj b/api/test/wfl/integration/executor_test.clj index 9781b1628..95f127ccd 100644 --- a/api/test/wfl/integration/executor_test.clj +++ b/api/test/wfl/integration/executor_test.clj @@ -168,13 +168,15 @@ (succeeded-workflow-from-submission submission-id)])})) ;; Workflow fetch mocks within update-workflow-statuses! -(defn ^:private mock-firecloud-get-running-workflow-update-status [_ submission-id workflow-id] +(defn ^:private mock-firecloud-get-running-workflow-update-status + [_ submission-id workflow-id & _] (let [workflow-base (get-in submission-base [submission-id :running])] (is (= (:workflow-id workflow-base) workflow-id) "Expecting to fetch and update status for running workflow") (assoc (mock-firecloud-get-workflow workflow-base) :status "Succeeded"))) -(defn ^:private mock-firecloud-get-known-workflow [_ submission-id workflow-id] +(defn ^:private mock-firecloud-get-known-workflow + [_ submission-id workflow-id & _] (if-let [workflow-base (->> (get submission-base submission-id) vals (filter #(= workflow-id (:workflow-id %))) @@ -328,7 +330,7 @@ #'firecloud/get-workflow mock-firecloud-get-known-workflow} #(let [workflows-to-retry (jdbc/with-db-transaction [tx (postgres/wfl-db-config)] - (executor/executor-workflows-by-filters tx executor {:status "Running"}))] + (executor/executor-workflows tx executor {:status "Running"}))] (is (== 1 (count workflows-to-retry)) "Should have one running workflow to retry.") (executor/executor-retry-workflows! workload workflows-to-retry))) @@ -395,7 +397,7 @@ (is (== 2 (stage/queue-length executor)) "The retried workflow should remain visible downstream") (jdbc/with-db-transaction [tx (postgres/wfl-db-config)] - (is (== 1 (count (executor/executor-workflows tx executor))) + (is (== 1 (count (executor/executor-workflows tx executor {}))) "The retried workflow should not be returned"))))) (deftest test-terra-executor-queue-length diff --git a/api/test/wfl/integration/firecloud_test.clj b/api/test/wfl/integration/firecloud_test.clj index 25b725e0f..89638ee52 100644 --- a/api/test/wfl/integration/firecloud_test.clj +++ b/api/test/wfl/integration/firecloud_test.clj @@ -118,8 +118,13 @@ (deftest test-get-workflow (using-assemble-refbased-workflow-bindings - (let [wf (firecloud/get-workflow workspace submission workflow)] - (is (= pipeline (:workflowName wf)))))) + (let [wf (firecloud/get-workflow workspace submission workflow) + wf-status (firecloud/get-workflow workspace submission workflow "status")] + (is (= pipeline (:workflowName wf))) + (is (= "Succeeded" (:status wf))) + (is (empty? (:workflowName wf-status)) + "Workflow fetch specifying include-key should have keys restricted") + (is (= "Succeeded" (:status wf-status)))))) (deftest test-get-workflow-outputs (using-assemble-refbased-workflow-bindings diff --git a/api/test/wfl/integration/modules/covid_test.clj b/api/test/wfl/integration/modules/covid_test.clj index cdd87b0bf..8caedb288 100644 --- a/api/test/wfl/integration/modules/covid_test.clj +++ b/api/test/wfl/integration/modules/covid_test.clj @@ -207,7 +207,7 @@ ;; Workflow fetch mocks within update-workflow-statuses! -(defn ^:private mock-workflow-keep-status [_ _ workflow-id] +(defn ^:private mock-workflow-keep-status [_ _ workflow-id & _] (is (not (= (:workflowId succeeded-workflow-mock) workflow-id)) "Successful workflow records should be filtered out before firecloud fetch") running-workflow-mock) diff --git a/api/test/wfl/unit/executor_test.clj b/api/test/wfl/unit/executor_test.clj index 1839984a0..927f4fd11 100644 --- a/api/test/wfl/unit/executor_test.clj +++ b/api/test/wfl/unit/executor_test.clj @@ -6,35 +6,35 @@ (:import [java.util UUID] [wfl.util UserException])) -(deftest test-terra-executor-workflows-by-filters-sql-params +(deftest test-terra-executor-workflows-sql-params (let [executor {:details "TerraExecutor_00000001"} submission (str (UUID/randomUUID)) status "Failed" filters {:submission submission :status status}] (letfn [(arg-count [sql] (-> sql frequencies (get \? 0)))] (testing "No filters" - (let [[sql & params] (#'executor/terra-executor-workflows-by-filters-sql-params + (let [[sql & params] (#'executor/terra-executor-workflows-sql-params executor {})] (is (== 0 (arg-count sql)) "No filters should yield no query arguments") (is (empty? params) "No filters should yield no parameters"))) (testing "Submission filter only" - (let [[sql & params] (#'executor/terra-executor-workflows-by-filters-sql-params + (let [[sql & params] (#'executor/terra-executor-workflows-sql-params executor (select-keys filters [:submission]))] (is (== 1 (arg-count sql)) "Single filter (submission) should yield 1 query argument") (is (= [submission] params) "Submission filter should yield lone submission parameter"))) (testing "Status filter only" - (let [[sql & params] (#'executor/terra-executor-workflows-by-filters-sql-params + (let [[sql & params] (#'executor/terra-executor-workflows-sql-params executor (select-keys filters [:status]))] (is (== 1 (arg-count sql)) "Single filter (status) should yield 1 query argument") (is (= [status] params) "Status filter should yield lone status parameter"))) (testing "Submission and status filters" - (let [[sql & params] (#'executor/terra-executor-workflows-by-filters-sql-params + (let [[sql & params] (#'executor/terra-executor-workflows-sql-params executor (select-keys filters [:submission :status]))] (is (== 2 (arg-count sql)) "Two filters (submission and status) should yield 2 query arguments") diff --git a/docs/md/assets/terra-method-configuration.png b/docs/md/assets/terra-method-configuration.png deleted file mode 100644 index ef1f0de74..000000000 Binary files a/docs/md/assets/terra-method-configuration.png and /dev/null differ diff --git a/docs/md/dev-logging.md b/docs/md/dev-logging.md index e6ecb7276..c90d46ba6 100644 --- a/docs/md/dev-logging.md +++ b/docs/md/dev-logging.md @@ -91,35 +91,6 @@ The result would be similar: The above change would allow all logs DEBUG and higher to be written, i.e. DEBUG, INFO, NOTICE, WARNING, ERROR, CRITICAL, ALERT, EMERGENCY. -The result will look something like this: -``` -{ - "level" : "INFO" -} -``` - -In order to change this level as desired would be done like so: -``` -curl -X POST http://localhost:3000/api/v1/logging_level?level=DEBUG \ - -H 'accept: application/json' \ - -H "authorization: Bearer "$(gcloud auth print-access-token) -``` - -The result would be similar: -``` -{ - "level" : "DEBUG" -} -``` - -The above change would allow all logs DEBUG and higher to be written, i.e. DEBUG, INFO, NOTICE, -WARNING, ERROR, CRITICAL, ALERT, EMERGENCY. - -Example: -```clojure -(binding [log/*logger* log/disabled-logger] - (log/info "This message will not be written")) -``` ## Testing Test for this can be found in `test/wfl/unit/logging_test.clj`. Currently, the tests check whether the logging methods produce json that includes the correct severity and message. diff --git a/docs/md/executor.md b/docs/md/executor.md index eaad56d22..9554d1c8e 100644 --- a/docs/md/executor.md +++ b/docs/md/executor.md @@ -84,8 +84,14 @@ Example fetch of a method configuration's version from the appropriate Firecloud instance (prod): ```shell -curl -X 'GET' \ - 'https://firecloud-orchestration.dsde-prod.broadinstitute.org/api/workspaces/emerge_prod/Arrays_test/method_configs/warp-pipelines/Arrays' \ +FIRECLOUD=https://firecloud-orchestration.dsde-prod.broadinstitute.org + +# Should be as they'd appear in a Terra UI URL path: +WORKSPACE=emerge_prod/Arrays_test +METHOD_CONFIG=warp-pipelines/Arrays + +curl -X GET \ + "$FIRECLOUD/api/workspaces/$WORKSPACE/method_configs/$METHOD_CONFIG" \ -H 'accept: */*' \ -H 'Authorization: Bearer '$(gcloud auth print-access-token) \ | jq .methodConfigVersion @@ -133,20 +139,14 @@ An executor is a `Queue` that satisfies the `Executor` protocol below: are parameterised types and the Source queue's parameterisation must be convertible to the Executor's.") (executor-workflows - ^IPersistentVector - [^Connection transaction ;; JDBC Connection - ^Executor executor ;; This executor instance - ] - "Use database `transaction` to return workflows created by the `executor`.") - (executor-workflows-by-filters ^IPersistentVector [^Connection transaction ;; JDBC Connection ^Executor executor ;; This executor instance - ^IPersistentVector filters ;; Workflow filters to match + ^IPersistentVector filters ;; Optional workflow filters to match ;; (ex. status, submission) ] "Use database `transaction` to return workflows created by the `executor` - matching the workflow `filters` (ex. status, submission).") + matching the optional workflow `filters` (ex. status, submission).") (executor-throw-if-invalid-retry-filters ;; Executed for side effects [^IPersistentHashmap workload ;; Workload for which a retry is requested diff --git a/docs/md/modules-covid.md b/docs/md/modules-covid.md index 68ebe1996..5619f573a 100644 --- a/docs/md/modules-covid.md +++ b/docs/md/modules-covid.md @@ -11,15 +11,15 @@ which includes a source, executor, and sink. The `covid` module supports the following API endpoints: -| Verb | Endpoint | Description | -|------|-------------------------------------|-----------------------------------------------------------------------| -| GET | `/api/v1/workload` | List all workloads, optionally filtering by uuid or project | -| GET | `/api/v1/workload/{uuid}/workflows` | List all workflows for workload `uuid`, filtering by supplied filters | -| POST | `/api/v1/workload/{uuid}/retry` | Retry workflows matching given filters in workload `uuid` | -| POST | `/api/v1/create` | Create a new workload | -| POST | `/api/v1/start` | Start a workload | -| POST | `/api/v1/stop` | Stop a running workload | -| POST | `/api/v1/exec` | Create and start (execute) a workload | +| Verb | Endpoint | Description | +|------|-------------------------------------|---------------------------------------------------------------------------------| +| GET | `/api/v1/workload` | List all workloads, optionally filtering by uuid or project | +| GET | `/api/v1/workload/{uuid}/workflows` | List all unretried workflows for workload `uuid`, filtering by supplied filters | +| POST | `/api/v1/workload/{uuid}/retry` | Retry unretried workflows matching given filters in workload `uuid` | +| POST | `/api/v1/create` | Create a new workload | +| POST | `/api/v1/start` | Start a workload | +| POST | `/api/v1/stop` | Stop a running workload | +| POST | `/api/v1/exec` | Create and start (execute) a workload | The life-cycle of a workload is a multi-stage process: @@ -156,33 +156,52 @@ The response is an array of [workload objects](#workload-response-format). Query WFL for all unretried workflows associated with workload `uuid`. -The response is a list of Firecloud-derived -[workflows](https://firecloud-orchestration.dsde-prod.broadinstitute.org/#/Submissions/workflowMetadata) -and their -[outputs](https://firecloud-orchestration.dsde-prod.broadinstitute.org/#/Submissions/workflowOutputsInSubmission) -when available (when the workflow has succeeded). +!!! warning "Note" + Fetching workflows in a workload without any other filters + may yield a large response and take a long time to process, + such as for long-running continuous workloads. === "Request" ``` - curl -X GET 'http://localhost:3000/api/v1/workload/8d67a71e-9afd-4fca-bcf6-a7a74984a0e8/workflows' \ + WFL=http://localhost:3000 + UUID=e8bc2c14-2396-469e-80fe-ebfed8d60a22 # workload UUID + + curl -X GET \ + "$WFL/api/v1/workload/$UUID/workflows" \ -H 'Authorization: Bearer '$(gcloud auth print-access-token) \ -H 'Accept: application/json' ``` +The response is a list of WFL workflow records, each of which tell us: + +- WFL submitted `methodConfiguration` in `workspace` with the + snapshot `reference` as its root entity type + +- The resulting submission had Terra `submission` UUID + +- The workflow had Terra `workflow` UUID and is processing the snapshot row + with `entity` as its row UUID + +- At WFL's last poll at `updated`, we noted that the workflow had `status` + +- Once a workflow has succeeded, `consumed` is populated with a timestamp + when WFL has "sunk" its outputs to their destination === "Response" ``` [ { - "inputs" : { - "biosample_to_genbank.docker" : "quay.io/broadinstitute/viral-phylo:2.1.19.1", - "instrument_model" : "Illumina NovaSeq 6000", - ... - }, - "uuid" : "53f70344-6f0f-47fb-adee-4e780fb3f19a", - "status" : "Failed", - "outputs" : { }, - "updated" : "2021-08-06T21:41:28Z" + "retry": null, + "workspace": "emerge_prod/Arrays_test_AD", + "updated": "2022-01-03T21:35:40Z", + "workflow": "99956c17-26af-4d50-8954-9de9ecc4f733", + "reference": "4de4b53c-3904-43f9-a155-9f2e2460eb42", + "status": "Aborted", + "id": 1, + "methodConfiguration": "emerge_prod/Arrays_TDR_GH-1560", + "consumed": null, + "submission": "fbb96180-f6a5-4895-a154-72d133e442db", + "entity": "d4ce2b14-49d6-4209-95ab-424e2edf1741" } ] ``` @@ -199,7 +218,13 @@ The response has the same format as when querying without filters. === "Request" ``` - curl -X GET 'http://localhost:3000/api/v1/workload/8d67a71e-9afd-4fca-bcf6-a7a74984a0e8/workflows?submission=14bffc69-6ce7-4615-b318-7ef1c457c894&status=Succeeded' \ + WFL=http://localhost:3000 + UUID=e8bc2c14-2396-469e-80fe-ebfed8d60a22 # workload UUID + SUBMISSION=fbb96180-f6a5-4895-a154-72d133e442db # Terra submission UUID + STATUS=Aborted # Cromwell workflow status + + curl -X GET \ + "$WFL/api/v1/workload/$UUID/workflows?submission=$SUBMISSION&status=$STATUS" \ -H 'Authorization: Bearer '$(gcloud auth print-access-token) \ -H 'Accept: application/json' ``` @@ -231,10 +256,15 @@ Further information found in general === "Request" ``` - curl -X POST "http://localhost:3000/api/v1/workload/e66c86b2-120d-4f7f-9c3a-b9eaadeb1919/retry" \ + WFL=http://localhost:3000 + UUID=e8bc2c14-2396-469e-80fe-ebfed8d60a22 # workload UUID + SUBMISSION=fbb96180-f6a5-4895-a154-72d133e442db # Terra submission UUID + + curl -X POST \ + "$WFL/api/v1/workload/$UUID/retry" \ -H 'Authorization: Bearer '$(gcloud auth print-access-token) \ -H "Content-Type: application/json" \ - -d '{ "submission": "14bffc69-6ce7-4615-b318-7ef1c457c894" }' + -d "{ \"submission\": \"$SUBMISSION\" }" ``` ### Create Workload diff --git a/docs/md/usage-retry.md b/docs/md/usage-retry.md index 4b8af6ca2..b42237485 100644 --- a/docs/md/usage-retry.md +++ b/docs/md/usage-retry.md @@ -20,7 +20,7 @@ UUID=0d307eb3-2b8e-419c-b687-8c08c84e2a0c # workload UUID SUBMISSION=14bffc69-6ce7-4615-b318-7ef1c457c894 # Terra submission UUID curl -X POST -H "$AUTH" $WFL/$UUID/retry \ - --data '{"submission":"$SUBMISSION"}' \ + --data "{\"submission\":\"$SUBMISSION\"}" \ | jq ``` diff --git a/version b/version index ac454c6a1..34a83616b 100644 --- a/version +++ b/version @@ -1 +1 @@ -0.12.0 +0.12.1