ToolsVanBox · mroosmalen · Oct 1, 2025 · Dec 30, 2024 · Jan 30, 2025 · Jan 30, 2025
diff --git a/configs/base.config b/configs/base.config
@@ -32,7 +32,18 @@ def check_max(obj, type) {
             println "   ### ERROR ###   Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
             return obj
         }
+    } else if (type == 'disk') {
+        try {
+            if (obj.compareTo(params.max_disk as nextflow.util.MemoryUnit) == 1)
+                return params.max_disk as nextflow.util.MemoryUnit
+            else
+                return obj
+        } catch (all) {
+            println "   ### ERROR ###   Max disk '${params.max_disk}' is not valid! Using default value: $obj"
+            return obj
+        }
     }
+
 }
 
 process {
@@ -89,18 +100,32 @@ process {
         time = { check_max( 2.h  * task.attempt, 'time'    ) }
         memory =  { check_max( 90.GB * task.attempt, 'memory' ) }
     }
+    withName: 'BWAMEM2_MEM' {
+        disk = 100.GB
+    }
+    withName: 'SAMTOOLS_SORT' {
+        disk = 150.GB
+    }
+    withName: 'SAMTOOLS_MERGE' {
+        disk = 60.GB
+    }
+    withName: 'GATK4_MARKDUPLICATES_SPARK' {
+        disk = 200.GB
+    }    
     withName: 'SAMTOOLS_MPILEUP' {
-        time = { check_max( 12.h  * task.attempt, 'time'    ) }        
+        time = { check_max( 12.h  * task.attempt, 'time'    ) }
+        disk = 250.GB
     }
     withName: 'FREEC_GERMLINE' {
         time = { check_max( 2.h  * task.attempt, 'time'    ) }
         memory =  { check_max( 80.GB * task.attempt, 'memory' ) }
         cpus = { check_max( 6     * task.attempt, 'cpus'    ) }
     }
-    withName: 'GRIDSS' {
-        time = { check_max( 2.h  * task.attempt, 'time'    ) }
-        memory =  { check_max( 80.GB * task.attempt, 'memory' ) }
-        cpus = { check_max( 8     * task.attempt, 'cpus'    ) }
+    withName: 'GRIDSS_GERMLINE' {
+        time = { check_max( 24.h  * task.attempt, 'time'    ) }
+        memory =  { check_max( 200.GB * task.attempt, 'memory' ) }
+        cpus = { check_max( 10     * task.attempt, 'cpus'    ) }
+        drive = 100.GB
     }
     withName: 'GATK4_MUTECT2' {
         cpus          = { check_max( 6     * task.attempt, 'cpus'    ) }

diff --git a/configs/nextflow.config b/configs/nextflow.config
@@ -64,14 +64,26 @@ profiles {
     process.executor = 'google-batch'
     executor.queueSize = 1000
     execute.pollInterval = '60sec'
+    executor.jobName = {
+        def safeName = task.name
+                    .replaceAll(/[^a-zA-Z0-9-]/, "-") 
+                    .take(30)
+                    .toLowerCase()
+        def safeTag = task.tag
+                    .replaceAll(/[^a-zA-Z0-9-]/, "-") 
+                    .take(12)
+                    .toLowerCase()
+	def safeHash = task.hash.take(8)        
+	return "${safeTag}-${safeName}-${safeHash}"
+    }    
 
-    google.batch.bootDiskSize = '333GB'
-
     google.region  = 'europe-west4'
     google.project = 'pmc-gcp-box-d-pip-development'
     google.location = 'europe-west4'
- 
+
     google.batch.spot = true
+    google.batch.autoRetryExitCodes = [104, 9, 50001, 50002, 50003, 50004, 50005, 50006]
+    google.batch.maxSpotAttempts = 10
     google.batch.usePrivateAddress = true
     google.batch.network =  'projects/pmc-vpc-res-private-20gx/global/networks/shared-vpc-res-priv-dev'
     google.batch.subnetwork = 'projects/pmc-vpc-res-private-20gx/regions/europe-west4/subnetworks/subnet-res-priv-dev'
@@ -99,4 +111,4 @@ timeline {
 
 }
 
-cleanup = true
+cleanup = true
diff --git a/configs/run-template.config b/configs/run-template.config
@@ -32,23 +32,23 @@ params {
         vcf_germline_short_variant_somatic_filtration = true
 
         // Somatic short variant discovery
-        bam_somatic_short_variant_discovery = true
-        vcf_somatic_short_variant_filtration = true
+        bam_somatic_short_variant_discovery = false // FALSE
+        vcf_somatic_short_variant_filtration = false // FALSE
 
 	// Short variant annotation
         vcf_short_variant_annotation = true
 
         // Copy number discovery
-        bam_germline_copy_number_discovery = false
-        bam_somatic_copy_number_discovery = false
-        bam_tumoronly_copy_number_discovery = false
+        bam_germline_copy_number_discovery = false // TRUE
+        bam_somatic_copy_number_discovery = false // FALSE
+        bam_tumoronly_copy_number_discovery = false // FALSE
 
         // Structural variant discovery
-        bam_germline_structural_variant_discovery = false
-        bam_somatic_structural_variant_discovery = false
-        bam_tumoronly_structural_variant_discovery = false
+        bam_germline_structural_variant_discovery = false // TRUE
+        bam_somatic_structural_variant_discovery = false // FALSE
+        bam_tumoronly_structural_variant_discovery = false //FALSE
 
-        vcf_structural_variant_filtration = false
+        vcf_somatic_structural_variant_filtration = true // TRUE
 
 	// HLA typing
 	bam_hla_type_calling = false

diff --git a/configs/settings.config b/configs/settings.config
@@ -412,7 +412,7 @@ process {
     }
 
     withName: 'SNPSIFT_SPLIT' {
-        ext.args         = { "-l 1000" }
+        ext.args         = { "-l 5000" }
         ext.prefix       = { }
     }
     withName: 'SNPSIFT_JOIN_.*' {

diff --git a/create_samplesheet_gcloud.sh b/create_samplesheet_gcloud.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+MYFOLDER=$1
+SAMPLE_TYPE='tumor'
+
+R1_FASTQS=($(gcloud storage ls ${MYFOLDER}/** | grep -P ".*_1.f.*q.gz"))
+R1_FASTQS+=($(gcloud storage ls ${MYFOLDER}/** | grep -P ".*_R1_.*.f.*q.gz"))
+R1_FASTQS+=($(gcloud storage ls ${MYFOLDER}/** | grep -P ".*_R1.f.*q.gz"))
+
+#R1_FASTQS=($(find ${MYFOLDER} -iname "*_1.f*q.gz"))
+#R1_FASTQS+=($(find ${MYFOLDER} -iname "*_R1_*.f*q.gz"))
+
+echo "sample,fastq_1,fastq_2,bam,bai,sample_type"
+
+for FASTQ_1 in ${R1_FASTQS[@]}; do 
+	BAM=""
+	BAI=""
+	SAMPLE=$( echo $(basename ${FASTQ_1}) | cut -f1 -d'_' | cut -f1 -d'.')
+        if [[ "${FASTQ_1}" == *"_R1_"* ]]; then
+		FASTQ_2=${FASTQ_1/_R1_/_R2_}
+	fi
+	if [[ "${FASTQ_1}" == *"_1.f"* ]]; then
+		FASTQ_2=${FASTQ_1/_1.f/_2.f}
+        fi
+        if [[ "${FASTQ_1}" == *"_R1.f"* ]]; then
+    		FASTQ_2=${FASTQ_1/_R1.f/_R2.f}
+    	fi
+	if gsutil -q stat ${FASTQ_2}; then
+		echo ${SAMPLE},${FASTQ_1},${FASTQ_2},${BAM},${BAI},${SAMPLE_TYPE}
+	else
+		echo "NO PAIRED FASTQ FILES FOUND FOR ${SAMPLE}"
+        fi
+done
+
+BAMS=($(gcloud storage ls ${MYFOLDER}/** | grep -P ".*.bam$"))
+#BAMS=($(find ${MYFOLDER} -iname "*bam"))
+
+for BAM in ${BAMS[@]}; do 
+	FASTQ_1=""
+	FASTQ_2=""
+	SAMPLE=$( echo $(basename ${BAM}) | cut -f1 -d'_' | cut -f1 -d'.')
+	BAI=${BAM}.bai
+
+	if ! gsutil -q stat ${BAI} ; then
+		BAI=${BAM/.bam/.bai}
+	fi
+	if ! gsutil -q stat ${BAI}; then
+		echo "NO BAI FILE FOUND FOR ${SAMPLE}"
+	else
+		echo ${SAMPLE},${FASTQ_1},${FASTQ_2},${BAM},${BAI},${SAMPLE_TYPE}
+	fi
+done
+
diff --git a/create_samplesheet_gcloud2.sh b/create_samplesheet_gcloud2.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+MYFOLDER=$1
+SAMPLE=$2
+SAMPLE_TYPE='tumor'
+
+R1_FASTQS=($(gcloud storage ls ${MYFOLDER}/** | grep -P "${SAMPLE}.*_1.f.*q.gz"))
+R1_FASTQS+=($(gcloud storage ls ${MYFOLDER}/** | grep -P "${SAMPLE}.*_R1_.*.f.*q.gz"))
+R1_FASTQS+=($(gcloud storage ls ${MYFOLDER}/** | grep -P "${SAMPLE}.*_R1.f.*q.gz"))
+
+#R1_FASTQS=($(find ${MYFOLDER} -iname "*_1.f*q.gz"))
+#R1_FASTQS+=($(find ${MYFOLDER} -iname "*_R1_*.f*q.gz"))
+
+echo "sample,fastq_1,fastq_2,bam,bai,sample_type"
+
+for FASTQ_1 in ${R1_FASTQS[@]}; do 
+	BAM=""
+	BAI=""
+	SAMPLE=$( echo $(basename ${FASTQ_1}) | cut -f1 -d'_' | cut -f1 -d'.')
+        if [[ "${FASTQ_1}" == *"_R1_"* ]]; then
+		FASTQ_2=${FASTQ_1/_R1_/_R2_}
+	fi
+	if [[ "${FASTQ_1}" == *"_1.f"* ]]; then
+		FASTQ_2=${FASTQ_1/_1.f/_2.f}
+        fi
+        if [[ "${FASTQ_1}" == *"_R1.f"* ]]; then
+    		FASTQ_2=${FASTQ_1/_R1.f/_R2.f}
+    	fi
+	if gsutil -q stat ${FASTQ_2}; then
+		echo ${SAMPLE},${FASTQ_1},${FASTQ_2},${BAM},${BAI},${SAMPLE_TYPE}
+	else
+		echo "NO PAIRED FASTQ FILES FOUND FOR ${SAMPLE}"
+        fi
+done
+
+BAMS=($(gcloud storage ls ${MYFOLDER}/** | grep -P ".*.bam$"))
+#BAMS=($(find ${MYFOLDER} -iname "*bam"))
+
+for BAM in ${BAMS[@]}; do 
+	FASTQ_1=""
+	FASTQ_2=""
+	SAMPLE=$( echo $(basename ${BAM}) | cut -f1 -d'_' | cut -f1 -d'.')
+	BAI=${BAM}.bai
+
+	if ! gsutil -q stat ${BAI} ; then
+		BAI=${BAM/.bam/.bai}
+	fi
+	if ! gsutil -q stat ${BAI}; then
+		echo "NO BAI FILE FOUND FOR ${SAMPLE}"
+	else
+		echo ${SAMPLE},${FASTQ_1},${FASTQ_2},${BAM},${BAI},${SAMPLE_TYPE}
+	fi
+done
+
diff --git a/main.nf b/main.nf
@@ -17,4 +17,4 @@ workflow {
     }
 
     ASAP()
-}
+}
diff --git a/modules/local/controlfreec/makebafplot/main.nf b/modules/local/controlfreec/makebafplot/main.nf
@@ -24,7 +24,7 @@ process CONTROLFREEC_MAKEBAFPLOT {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        R: \$(echo \$(R --version 2>&1) ) |  grep -oP "R version .+ --" | cut -f 3 -d' '
+        R: \$(echo \$(R --version 2>&1) |  grep -oP "R version .+ --" | cut -f 3 -d' ' )
     END_VERSIONS
     """
 
@@ -35,7 +35,7 @@ process CONTROLFREEC_MAKEBAFPLOT {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        R: \$(echo \$(R --version 2>&1) ) |  grep -oP "R version .+ --" | cut -f 3 -d' '
+	R: \$(echo \$(R --version 2>&1) |  grep -oP "R version .+ --" | cut -f 3 -d' ' )
     END_VERSIONS
     """
 }
diff --git a/modules/local/controlfreec/makekaryotype/main.nf b/modules/local/controlfreec/makekaryotype/main.nf
@@ -27,7 +27,7 @@ process CONTROLFREEC_MAKEKARYOTYPE {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        R: \$(echo \$(R --version 2>&1) ) |  grep -oP "R version .+ --" | cut -f 3 -d' '
+        R: \$(echo \$(R --version 2>&1) |  grep -oP "R version .+ --" | cut -f 3 -d' ' )
     END_VERSIONS
     """
 
@@ -38,7 +38,7 @@ process CONTROLFREEC_MAKEKARYOTYPE {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        R: \$(echo \$(R --version 2>&1) ) |  grep -oP "R version .+ --" | cut -f 3 -d' '
+	R: \$(echo \$(R --version 2>&1) |  grep -oP "R version .+ --" | cut -f 3 -d' ' )
     END_VERSIONS
     """
 }
diff --git a/modules/local/fingerprint/heatmap/main.nf b/modules/local/fingerprint/heatmap/main.nf
@@ -27,7 +27,7 @@ process FINGERPRINT_HEATMAP {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        R: \$(echo \$(R --version 2>&1) ) |  grep -oP "R version .+ --" | cut -f 3 -d' '
+        R: \$(echo \$(R --version 2>&1) |  grep -oP "R version .+ --" | cut -f 3 -d' ' )
     END_VERSIONS
     """
 
@@ -38,7 +38,7 @@ process FINGERPRINT_HEATMAP {
 
     cat <<-END_VERSIONS > versions.yml
     "${task.process}":
-        R: \$(echo \$(R --version 2>&1) ) |  grep -oP "R version .+ --" | cut -f 3 -d' '
+	R: \$(echo \$(R --version 2>&1) |  grep -oP "R version .+ --" | cut -f 3 -d' ' )
     END_VERSIONS
     """
 

diff --git a/modules/nf-core/gatk4/applybqsr/main.nf b/modules/nf-core/gatk4/applybqsr/main.nf
@@ -5,7 +5,8 @@ process GATK4_APPLYBQSR {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(input), path(input_index), path(bqsr_table), path(intervals)

diff --git a/modules/nf-core/gatk4/applyvqsr/main.nf b/modules/nf-core/gatk4/applyvqsr/main.nf
@@ -5,7 +5,8 @@ process GATK4_APPLYVQSR {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(vcf), path(vcf_tbi), path(recal), path(recal_index), path(tranches)

diff --git a/modules/nf-core/gatk4/baserecalibrator/main.nf b/modules/nf-core/gatk4/baserecalibrator/main.nf
@@ -5,7 +5,8 @@ process GATK4_BASERECALIBRATOR {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(input), path(input_index), path(intervals)

diff --git a/modules/nf-core/gatk4/calculatecontamination/main.nf b/modules/nf-core/gatk4/calculatecontamination/main.nf
@@ -5,7 +5,8 @@ process GATK4_CALCULATECONTAMINATION {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(pileup), path(matched)

diff --git a/modules/nf-core/gatk4/combinegvcfs/main.nf b/modules/nf-core/gatk4/combinegvcfs/main.nf
@@ -5,7 +5,8 @@ process GATK4_COMBINEGVCFS {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(vcf), path(vcf_idx)

diff --git a/modules/nf-core/gatk4/filtermutectcalls/main.nf b/modules/nf-core/gatk4/filtermutectcalls/main.nf
@@ -5,7 +5,8 @@ process GATK4_FILTERMUTECTCALLS {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(vcf), path(vcf_tbi), path(stats), path(orientationbias), path(segmentation), path(table), val(estimate)

diff --git a/modules/nf-core/gatk4/gatherpileupsummaries/main.nf b/modules/nf-core/gatk4/gatherpileupsummaries/main.nf
@@ -5,7 +5,8 @@ process GATK4_GATHERPILEUPSUMMARIES {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
 
     input:

diff --git a/modules/nf-core/gatk4/genomicsdbimport/main.nf b/modules/nf-core/gatk4/genomicsdbimport/main.nf
@@ -5,7 +5,8 @@ process GATK4_GENOMICSDBIMPORT {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(vcf), path(tbi), path(interval_file), val(interval_value), path(wspace)

diff --git a/modules/nf-core/gatk4/genotypegvcfs/main.nf b/modules/nf-core/gatk4/genotypegvcfs/main.nf
@@ -5,7 +5,8 @@ process GATK4_GENOTYPEGVCFS {
     // conda "bioconda::gatk4=4.4.0.0"
     container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ?
         'https://depot.galaxyproject.org/singularity/gatk4:4.4.0.0--py36hdfd78af_0':
-        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
+        'europe-west4-docker.pkg.dev/pmc-gcp-box-d-pip-development/pipeline-containers/gatk4@sha256:573f43fe59145d65106e5b2647d5c0f1662820716234a04104dfe1c91ebfa4be' }"
+//        'biocontainers/gatk4:4.4.0.0--py36hdfd78af_0' }"
 
     input:
     tuple val(meta), path(gvcf), path(gvcf_index), path(intervals), path(intervals_index)
-Original file line number
+Diff line change
@@ Expand Up / @@ -17,4 +17,4 @@ workflow { @@
         }
         ASAP()
-    }
+    }