replace test by test_cache and add test with non obfuscated files

conf/test.config

@@ -0,0 +1,114 @@
+/*
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Nextflow config file for running minimal tests
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    Defines input files and everything required to run a fast and simple pipeline test.
+
+    Use as follows:
+        nextflow run nf-core/sarek -profile test,<extra_test_profile>,<docker/singularity> --outdir <OUTDIR>
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+*/
+
+params {
+    config_profile_name        = 'Test profile'
+    config_profile_description = 'Minimal test dataset to check pipeline function'
+
+    // Limit resources so that this can run on GitHub Actions
+    max_cpus   = 2
+    max_memory = '6.5GB'
+    max_time   = '8.h'
+
+    // Input data
+    input = "${projectDir}/tests/csv/3.0/fastq_single.csv"
+
+    // Small reference genome
+    genome            = null
+    igenomes_ignore   = true
+    dbsnp             = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/dbsnp_146.hg38.vcf.gz"
+    fasta             = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.fasta"
+    germline_resource = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/gnomAD.r2.1.1.vcf.gz"
+    intervals         = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/genome.interval_list"
+    known_indels      = "https://raw.githubusercontent.com/nf-core/test-datasets/modules/data/genomics/homo_sapiens/genome/vcf/mills_and_1000G.indels.vcf.gz"
+    snpeff_db         = 105
+    snpeff_genome     = 'WBcel235'
+    snpeff_version    = '5.1'
+    vep_cache_version = 106
+    vep_genome        = 'WBcel235'
+    vep_species       = 'caenorhabditis_elegans'
+    vep_version       = '106.1'
+
+    // default params
+    split_fastq       = 0         // no FASTQ splitting
+    tools             = 'strelka' // Variant calling with Strelka
+
+    // Ignore params that will throw warning through params validation
+    schema_ignore_params = 'genomes,snpeff_version,vep_version'
+}
+
+process {
+    withName:'.*:FREEC_SOMATIC'{
+        ext.args = {
+            [
+                "sample":[
+                    inputformat: "pileup",
+                    mateorientation: "FR"
+                ],
+                "general":[
+                    bedgraphoutput: "TRUE",
+                    noisydata: "TRUE",
+                    minexpectedgc: "0",
+                    readcountthreshold: "1",
+                    sex: meta.sex,
+                    window: "10",
+                    ],
+                "control":[
+                    inputformat: "pileup",
+                    mateorientation: "FR"
+                ]
+            ]
+        }
+    }
+
+    if (params.tools && params.tools.split(',').contains('mutect2')) {
+        withName: '.*MUTECT2_PAIRED'{
+            //sample name from when the test data was generated
+            ext.args = { "--f1r2-tar-gz ${task.ext.prefix}.f1r2.tar.gz --normal-sample normal " }
+        }
+    }
+
+    withName: '.*:FILTERVARIANTTRANCHES'{
+        ext.args = { "--info-key CNN_1D --indel-tranche 0" }
+    }
+}
+
+// Enable container engines/virtualisation envs for CI testing
+// only works when specified with the profile ENV
+// otherwise tests can be done with the regular provided profiles
+if (System.getenv('PROFILE')) {
+    if ("$PROFILE" == "conda") {
+        conda.createTimeout                      = "120 min"
+        conda.enabled                            = true
+        charliecloud.enabled                     = false
+        docker.enabled                           = false
+        podman.enabled                           = false
+        shifter.enabled                          = false
+        singularity.enabled                      = false
+    } else if ("$PROFILE" == "docker") {
+        conda.enabled                            = false
+        docker.enabled                           = true
+        docker.userEmulation                     = { params.use_gatk_spark ? false : true }.call()
+        charliecloud.enabled                     = false
+        podman.enabled                           = false
+        shifter.enabled                          = false
+        singularity.enabled                      = false
+    } else if ("$PROFILE" == "singularity") {
+        conda.enabled                            = false
+        params.singularity_pull_docker_container = false
+        singularity.autoMounts                   = true
+        singularity.enabled                      = true
+        charliecloud.enabled                     = false
+        docker.enabled                           = false
+        podman.enabled                           = false
+        shifter.enabled                          = false
+    }
+}

nextflow.config

@@ -239,35 +239,36 @@ profiles {
         includeConfig 'conf/public_aws_ecr.config'
     }
     // Basic test profile for CI
-    test                       { includeConfig 'conf/test/test.config' }
+    test                       { includeConfig 'conf/test.config' }
+    test_cache                 { includeConfig 'conf/test/cache.config' }
     // Extra test profiles for full tests on AWS
-    test_full                  { includeConfig 'conf/test/test_full.config' }
-    test_full_germline         { includeConfig 'conf/test/test_full_germline.config' }
+    test_full                  { includeConfig 'conf/test_full.config' }
+    test_full_germline         { includeConfig 'conf/test_full_germline.config' }
     // Extra test profiles for more complete CI
-    alignment_to_fastq         { includeConfig 'conf/test/test_alignment_to_fastq.config' }
-    annotation                 { includeConfig 'conf/test/test_annotation.config' }
-    markduplicates_bam         { includeConfig 'conf/test/test_markduplicates_bam.config' }
-    markduplicates_cram        { includeConfig 'conf/test/test_markduplicates_cram.config' }
-    no_intervals               { includeConfig 'conf/test/test_no_intervals.config' }
-    pair                       { includeConfig 'conf/test/test_pair.config' }
-    prepare_recalibration_bam  { includeConfig 'conf/test/test_prepare_recalibration_bam.config' }
-    prepare_recalibration_cram { includeConfig 'conf/test/test_prepare_recalibration_cram.config' }
-    recalibrate_bam            { includeConfig 'conf/test/test_recalibrate_bam.config' }
-    recalibrate_cram           { includeConfig 'conf/test/test_recalibrate_cram.config' }
-    save_bam_mapped            { includeConfig 'conf/test/test_save_bam_mapped.config' }
-    skip_bqsr                  { includeConfig 'conf/test/test_skip_bqsr.config' }
-    skip_markduplicates        { includeConfig 'conf/test/test_skip_markduplicates.config' }
-    split_fastq                { includeConfig 'conf/test/test_split_fastq.config' }
-    targeted                   { includeConfig 'conf/test/test_targeted.config' }
-    tools                      { includeConfig 'conf/test/test_tools.config' }
-    tools_germline             { includeConfig 'conf/test/test_tools_germline.config' }
-    tools_somatic              { includeConfig 'conf/test/test_tools_somatic.config' }
-    tools_somatic_ascat        { includeConfig 'conf/test/test_tools_somatic_ascat.config' }
-    tools_tumoronly            { includeConfig 'conf/test/test_tools_tumoronly.config' }
-    trimming                   { includeConfig 'conf/test/test_trimming.config' }
-    umi                        { includeConfig 'conf/test/test_umi.config' }
-    use_gatk_spark             { includeConfig 'conf/test/test_use_gatk_spark.config' }
-    variantcalling_channels    { includeConfig 'conf/test/test_variantcalling_channels.config' }
+    alignment_to_fastq         { includeConfig 'conf/test/alignment_to_fastq.config' }
+    annotation                 { includeConfig 'conf/test/annotation.config' }
+    markduplicates_bam         { includeConfig 'conf/test/markduplicates_bam.config' }
+    markduplicates_cram        { includeConfig 'conf/test/markduplicates_cram.config' }
+    no_intervals               { includeConfig 'conf/test/no_intervals.config' }
+    pair                       { includeConfig 'conf/test/pair.config' }
+    prepare_recalibration_bam  { includeConfig 'conf/test/prepare_recalibration_bam.config' }
+    prepare_recalibration_cram { includeConfig 'conf/test/prepare_recalibration_cram.config' }
+    recalibrate_bam            { includeConfig 'conf/test/recalibrate_bam.config' }
+    recalibrate_cram           { includeConfig 'conf/test/recalibrate_cram.config' }
+    save_bam_mapped            { includeConfig 'conf/test/save_bam_mapped.config' }
+    skip_bqsr                  { includeConfig 'conf/test/skip_bqsr.config' }
+    skip_markduplicates        { includeConfig 'conf/test/skip_markduplicates.config' }
+    split_fastq                { includeConfig 'conf/test/split_fastq.config' }
+    targeted                   { includeConfig 'conf/test/targeted.config' }
+    tools                      { includeConfig 'conf/test/tools.config' }
+    tools_germline             { includeConfig 'conf/test/tools_germline.config' }
+    tools_somatic              { includeConfig 'conf/test/tools_somatic.config' }
+    tools_somatic_ascat        { includeConfig 'conf/test/tools_somatic_ascat.config' }
+    tools_tumoronly            { includeConfig 'conf/test/tools_tumoronly.config' }
+    trimming                   { includeConfig 'conf/test/trimming.config' }
+    umi                        { includeConfig 'conf/test/umi.config' }
+    use_gatk_spark             { includeConfig 'conf/test/use_gatk_spark.config' }
+    variantcalling_channels    { includeConfig 'conf/test/variantcalling_channels.config' }
 }
 
 // Load igenomes.config if required

tests/test_aligner_bwamem.yml

@@ -1,5 +1,5 @@
 - name: Run bwamem
-  command: nextflow run main.nf -profile test --aligner bwa-mem --save_reference --outdir results
+  command: nextflow run main.nf -profile test_cache --aligner bwa-mem --save_reference --outdir results
   tags:
     - aligner
     - bwamem
@@ -74,7 +74,7 @@
     - path: results/reports/samtools/test/test.recal.cram.stats
     # conda changes md5sums for test
 - name: Build only index with bwa
-  command: nextflow run main.nf -profile test --build_only_index --input=false --outdir results
+  command: nextflow run main.nf -profile test_cache --build_only_index --input=false --outdir results
   tags:
     - aligner
     - build_only_index

tests/test_aligner_bwamem2.yml

@@ -1,5 +1,5 @@
 - name: Run bwamem2
-  command: nextflow run main.nf -profile test --aligner bwa-mem2 --save_reference --outdir results
+  command: nextflow run main.nf -profile test_cache --aligner bwa-mem2 --save_reference --outdir results
   tags:
     - aligner
     - bwamem2
@@ -74,7 +74,7 @@
     - path: results/reports/samtools/test/test.recal.cram.stats
     # conda changes md5sums for test
 - name: Build only index with bwa-mem2
-  command: nextflow run main.nf -profile test --build_only_index --aligner bwa-mem2 --input=false --outdir results
+  command: nextflow run main.nf -profile test_cache --build_only_index --aligner bwa-mem2 --input=false --outdir results
   tags:
     - aligner
     - build_only_index

tests/test_aligner_dragmap.yml

@@ -1,5 +1,5 @@
 - name: Run dragmap
-  command: nextflow run main.nf -profile test --aligner dragmap --save_reference --outdir results
+  command: nextflow run main.nf -profile test_cache --aligner dragmap --save_reference --outdir results
   tags:
     - aligner
     - dragmap
@@ -87,7 +87,7 @@
     - path: results/reports/samtools/test/test.recal.cram.stats
     # conda changes md5sums for test
 - name: Build only index with dragmap
-  command: nextflow run main.nf -profile test --build_only_index --aligner dragmap --input=false --outdir results
+  command: nextflow run main.nf -profile test_cache --build_only_index --aligner dragmap --input=false --outdir results
   tags:
     - aligner
     - build_only_index

tests/test_alignment_to_fastq.yml

@@ -1,5 +1,5 @@
 - name: Run alignment to fastq and then remap on bam files
-  command: nextflow run main.nf -profile test,alignment_to_fastq --outdir results
+  command: nextflow run main.nf -profile test_cache,alignment_to_fastq --outdir results
   tags:
     - alignment_to_fastq
     - input_bam

tests/test_annotation_cache.yml

@@ -1,5 +1,5 @@
 - name: Only download annotation cache
-  command: nextflow run main.nf -profile test,annotation --tools merge --download_cache --input=false --build_only_index --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools merge --download_cache --input=false --build_only_index --outdir results
   tags:
     - annotation
     - cache
@@ -10,7 +10,7 @@
     - path: results/annotation
       should_exist: false
 - name: Download annotation cache and annotate using snpEff and VEP
-  command: nextflow run main.nf -profile test,annotation --tools snpeff,vep --download_cache --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools snpeff,vep --download_cache --outdir results
   tags:
     - annotation
     - cache

tests/test_annotation_merge.yml

@@ -1,5 +1,5 @@
 - name: Run snpEff followed by VEP
-  command: nextflow run main.nf -profile test,annotation --tools merge --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools merge --outdir results
   tags:
     - annotation
     - merge
@@ -28,7 +28,7 @@
     - path: results/reports/EnsemblVEP/test/test_VEP.ann.summary.html
       should_exist: false
 - name: Run VEP and snpEff followed by VEP
-  command: nextflow run main.nf -profile test,annotation --tools merge,snpeff,vep --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools merge,snpeff,vep --outdir results
   tags:
     - annotation
     - merge

tests/test_annotation_snpeff.yml

@@ -1,5 +1,5 @@
 - name: Run snpEff
-  command: nextflow run main.nf -profile test,annotation --tools snpeff --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools snpeff --outdir results
   tags:
     - annotation
     - snpeff

tests/test_annotation_vep.yml

@@ -1,5 +1,5 @@
 - name: Run VEP
-  command: nextflow run main.nf -profile test,annotation --tools vep --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools vep --outdir results
   tags:
     - annotation
     - vep
@@ -12,7 +12,7 @@
     - path: results/reports/EnsemblVEP/test/test_VEP.ann.summary.html
       contains: ["test.vcf.gz</td></tr><tr><td>Output file</td><td>test_VEP.ann.vcf.gz</td></tr>"]
 - name: Run VEP with fasta
-  command: nextflow run main.nf -profile test,annotation --tools vep --vep_include_fasta --outdir results
+  command: nextflow run main.nf -profile test_cache,annotation --tools vep --vep_include_fasta --outdir results
   tags:
     - annotation
     - vep

tests/test_cnvkit.yml

@@ -1,5 +1,5 @@
 - name: Run variant calling on somatic samples with cnvkit
-  command: nextflow run main.nf -profile test,tools_somatic --tools cnvkit --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_somatic --tools cnvkit --outdir results
   tags:
     - cnvkit
     - copy_number_calling
@@ -58,7 +58,7 @@
     - path: results/cnvkit
       should_exist: false
 - name: Run variant calling on tumor_only sample with cnvkit
-  command: nextflow run main.nf -profile test,tools_tumoronly --tools cnvkit --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools cnvkit --outdir results
   tags:
     - cnvkit
     - copy_number_calling
@@ -89,7 +89,7 @@
     - path: results/cnvkit
       should_exist: false
 - name: Run variant calling on germline sample with cnvkit
-  command: nextflow run main.nf -profile test,tools_germline --tools cnvkit --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_germline --tools cnvkit --outdir results
   tags:
     - cnvkit
     - copy_number_calling
@@ -122,7 +122,7 @@
     - path: results/cnvkit
       should_exist: false
 - name: Run variant calling on somatic samples with cnvkit and skip variant calling on matched normal
-  command: nextflow run main.nf -profile test,tools_somatic --tools cnvkit --only_paired_variant_calling --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_somatic --tools cnvkit --only_paired_variant_calling --outdir results
   tags:
     - cnvkit
     - somatic

tests/test_concat_germline_vcfs.yml

@@ -1,5 +1,5 @@
 - name: Run all germline variant callers and check for existence of concatenated vcf-files
-  command: nextflow run main.nf -profile test --input ./tests/csv/3.0/mapped_joint_bam.csv --concatenate_vcfs --tools deepvariant,freebayes,haplotypecaller,manta,mpileup,strelka,tiddit --step variant_calling --outdir results
+  command: nextflow run main.nf -profile test_cache --input ./tests/csv/3.0/mapped_joint_bam.csv --concatenate_vcfs --tools deepvariant,freebayes,haplotypecaller,manta,mpileup,strelka,tiddit --step variant_calling --outdir results
   tags:
     - concatenate_vcfs
   files:

tests/test_controlfreec.yml

@@ -1,5 +1,5 @@
 - name: Run variant calling on somatic samples with controlfreec
-  command: nextflow run main.nf -profile test,tools_somatic --tools controlfreec --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_somatic --tools controlfreec --outdir results
   tags:
     - controlfreec
     - somatic
@@ -53,7 +53,7 @@
     - path: results/cnvkit
       should_exist: false
 - name: Run variant calling on somatic samples with controlfreec without intervals
-  command: nextflow run main.nf -profile test,tools_somatic --tools controlfreec --no_intervals -stub-run --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_somatic --tools controlfreec --no_intervals -stub-run --outdir results
   tags:
     - controlfreec
     - no_intervals
@@ -106,7 +106,7 @@
     - path: results/mpileup
       should_exist: false
 - name: Run variant calling on tumor_only sample with controlfreec
-  command: nextflow run main.nf -profile test,tools_tumoronly --tools controlfreec -stub-run --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_tumoronly --tools controlfreec -stub-run --outdir results
   tags:
     - controlfreec
     - tumor_only

tests/test_deepvariant.yml

@@ -1,5 +1,5 @@
 - name: Run variant calling on germline sample with deepvariant
-  command: nextflow run main.nf -profile test,tools_germline --tools deepvariant --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_germline --tools deepvariant --outdir results
   tags:
     - deepvariant
     - germline
@@ -25,7 +25,7 @@
     - path: results/deepvariant
       should_exist: false
 - name: Run variant calling on germline sample with deepvariant without intervals
-  command: nextflow run main.nf -profile test,tools_germline --tools deepvariant --no_intervals --outdir results
+  command: nextflow run main.nf -profile test_cache,tools_germline --tools deepvariant --no_intervals --outdir results
   tags:
     - deepvariant
     - germline

tests/test_fastp.yml

@@ -1,5 +1,5 @@
 - name: Run trimming pipeline
-  command: nextflow run main.nf -profile test,trimming --save_trimmed --outdir results
+  command: nextflow run main.nf -profile test_cache,trimming --save_trimmed --outdir results
   tags:
     - fastp
     - preprocessing
@@ -55,7 +55,7 @@
     - path: results/reports/samtools/test/test.recal.cram.stats
     # conda changes md5sums for test
 - name: Run split fastq module
-  command: nextflow run main.nf -profile test,split_fastq --outdir results
+  command: nextflow run main.nf -profile test_cache,split_fastq --outdir results
   tags:
     - fastp
     - preprocessing