Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[GH-1061] don't send back defaults when user specifies reference_fasta_prefix #221

Merged
merged 2 commits into from
Nov 13, 2020
Merged

[GH-1061] don't send back defaults when user specifies reference_fasta_prefix #221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4f4caf3
don't send back defaults when user specifies reference_fasta_prefix
ehigham Nov 12, 2020
5125756
update exomes docs
ehigham Nov 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 11 additions & 10 deletions api/src/wfl/module/wgs.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,20 +47,19 @@
{:cram_ref_fasta cram_ref_fasta
:cram_ref_fasta_index (str cram_ref_fasta ".fai")}))

(defn make-references
(def ^:private default-references
"HG38 reference, calling interval, and contamination files."
[prefix]
(let [hg38 "gs://gcp-public-data--broad-references/hg38/v0/"]
(merge references/contamination-sites
(references/hg38-genome-references prefix)
(references/hg38-genome-references nil)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could we (def hg38-genome-references …) and just merge it as data?

{:calling_interval_list
(str hg38 "wgs_calling_regions.hg38.interval_list")})))

(def hack-task-level-values
"Hack to overload task-level values for wgs pipeline."
(let [hg38 "gs://gcp-public-data--broad-references/hg38/v0/"]
{:wgs_coverage_interval_list
(str hg38 "wgs_coverage_regions.hg38.interval_list")}))
(str hg38 "wgs_coverage_regions.hg38.interval_list")}))

(defn env-inputs
"Genome inputs for ENVIRONMENT that do not depend on the input file."
Expand All @@ -74,11 +73,12 @@
:scatter_settings {:haplotype_scatter_count 10
:break_bands_at_multiples_of 100000}}))

(defn ^:private normalize-references [inputs]
(update inputs :references
#(util/deep-merge
(-> inputs :reference_fasta_prefix make-references)
%)))
(defn ^:private normalize-reference-fasta [inputs]
(if-let [prefix (:reference_fasta_prefix inputs)]
(-> (update-in inputs [:references :reference_fasta]
#(util/deep-merge (references/reference_fasta prefix) %))
(dissoc :reference_fasta_prefix))
inputs))

(defn ^:private make-inputs-to-save
"Return inputs for reprocessing IN-GS into OUT-GS."
Expand All @@ -98,6 +98,7 @@
"Return inputs for reprocessing IN-GS into OUT-GS in ENVIRONMENT."
[environment workflow-inputs]
(-> (util/deep-merge cram-ref hack-task-level-values)
(util/deep-merge {:references default-references})
(util/deep-merge (env-inputs environment))
(util/deep-merge workflow-inputs)
(util/prefix-keys (keyword pipeline))))
Expand Down Expand Up @@ -128,7 +129,7 @@
#(json/write-str (util/deep-merge (:options common) %)))
(update :inputs
#(json/write-str
(normalize-references
(normalize-reference-fasta
(util/deep-merge
(:inputs common)
(make-inputs-to-save output %)))))))]
Expand Down
18 changes: 3 additions & 15 deletions api/src/wfl/module/xx.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
[wfl.module.batch :as batch]
[wfl.references :as references]
[wfl.service.gcs :as gcs]
[wfl.service.postgres :as postgres]
[wfl.service.cromwell :as cromwell]
[wfl.util :as util]
[wfl.wdl :as wdl]
Expand Down Expand Up @@ -69,7 +68,8 @@
(defn ^:private cromwellify-workflow-inputs [environment {:keys [inputs]}]
(-> (env/stuff environment)
(select-keys [:google_account_vault_path :vault_token_path])
(merge inputs)
(util/deep-merge workflow-defaults)
(util/deep-merge inputs)
(util/prefix-keys (keyword pipeline))))

;; visible for testing
Expand Down Expand Up @@ -129,18 +129,6 @@
(jdbc/update! tx :workload {:started now} ["id = ?" id]))
(workloads/load-workload-for-id tx id)))

(defmethod workloads/load-workload-impl
pipeline
[tx {:keys [items] :as workload}]
(letfn [(unnilify [m] (into {} (filter second m)))
(load-inputs [m]
(update m :inputs
#(util/deep-merge workflow-defaults (util/parse-json %))))
(load-options [m] (update m :options (fnil util/parse-json "null")))]
(->> (postgres/get-table tx items)
(mapv (comp unnilify load-inputs load-options))
(assoc workload :workflows)
unnilify)))

(defoverload workloads/create-workload! pipeline create-xx-workload!)
(defoverload workloads/start-workload! pipeline start-xx-workload!)
(defoverload workloads/load-workload-impl pipeline batch/load-batch-workload-impl)
11 changes: 8 additions & 3 deletions api/test/wfl/integration/modules/wgs_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,9 @@
(is (= reference-fasta (references/reference_fasta prefix))))
(go! [inputs]
(verify-reference-fasta
(get-in inputs [:references :reference_fasta])))]
(get-in inputs [:references :reference_fasta]))
(is (empty? (-> inputs :references (dissoc :reference_fasta))))
(is (util/absent? inputs :reference_fasta_prefix)))]
(run! (comp go! :inputs)
(-> (make-wgs-workload-request)
(assoc-in [:common :inputs] {:reference_fasta_prefix prefix})
Expand All @@ -42,7 +44,9 @@
(is (= reference-fasta (references/reference_fasta prefix))))
(go! [inputs]
(verify-reference-fasta
(get-in inputs [:references :reference_fasta])))]
(get-in inputs [:references :reference_fasta]))
(is (empty? (-> inputs :references (dissoc :reference_fasta))))
(is (util/absent? inputs :reference_fasta_prefix)))]
(run! (comp go! :inputs)
(-> (make-wgs-workload-request)
(assoc-in [:common :inputs] {:reference_fasta_prefix "gs://ignore/this/ref-fasta"})
Expand Down Expand Up @@ -110,7 +114,8 @@
(verify-workflow-inputs [inputs]
(is (:supports_common_inputs inputs))
(is (:supports_inputs inputs))
(is (:overwritten inputs)))
(is (:overwritten inputs))
(is (not-empty (-> inputs :references (dissoc :reference_fasta)))))
(verify-submitted-inputs [_ _ _ inputs _ _]
(is (every? #(prefixed? :ExternalWholeGenomeReprocessing %) (keys inputs)))
(verify-workflow-inputs (into {} (map strip-prefix inputs)))
Expand Down
Loading