Fix publishing

conf/modules/bwa_index.config

@@ -0,0 +1,12 @@
+process {
+    withName: BWA_INDEX {
+        publishDir = [
+            [
+                path: { "${params.outdir}/${params.aligner}/reference_genome" },
+                mode: params.publish_dir_mode,
+                pattern: "*",
+                enabled: params.save_reference
+            ]
+        ]
+    }
+}

conf/modules/picard_addorreplacereadgroups.config

@@ -1,5 +1,9 @@
 process {
     withName: PICARD_ADDORREPLACEREADGROUPS {
         ext.args = "--RGID 1 --RGLB lib1 --RGPL illumina --RGPU unit1 --RGSM sample1"
+        // Don't publish intermediate BAMs - these are only used as input to PICARD_MARKDUPLICATES
+        publishDir = [
+            enabled: false
+        ]
     }
 }

conf/modules/samtools_idxstats.config

@@ -0,0 +1,11 @@
+process {
+    withName: SAMTOOLS_IDXSTATS {
+        publishDir = [
+            [
+                path: { "${params.outdir}/${params.aligner}/alignments/samtools_stats/" },
+                mode: params.publish_dir_mode,
+                pattern: "*.idxstats"
+            ]
+        ]
+    }
+}

conf/modules/samtools_index.config

@@ -6,13 +6,13 @@ process {
                 path: { "${params.outdir}/${params.aligner}/deduplicated/" },
                 mode: params.publish_dir_mode,
                 pattern: "*.bai",
-                enabled: !params.skip_deduplication
+                enabled: !(params.skip_deduplication || params.rrbs)
             ],
             [
                 path: { "${params.outdir}/${params.aligner}/alignments/" },
                 mode: params.publish_dir_mode,
                 pattern: "*.bai",
-                enabled: params.skip_deduplication
+                enabled: params.skip_deduplication || params.rrbs
             ]
         ]
     }

conf/modules/samtools_sort.config

@@ -6,7 +6,7 @@ process {
                 path: { "${params.outdir}/${params.aligner}/alignments/" },
                 mode: params.publish_dir_mode,
                 pattern: "*.sorted.bam",
-                enabled: params.skip_deduplication || params.save_align_intermeds
+                enabled: params.skip_deduplication || params.rrbs || params.save_align_intermeds
             ]
         ]
     }

conf/subworkflows/fasta_index_methylseq.config

@@ -2,3 +2,4 @@ includeConfig "../modules/gunzip.config"
 includeConfig "../modules/bismark_genomepreparation.config"
 includeConfig "../modules/samtools_faidx.config"
 includeConfig "../modules/bwameth_index.config"
+includeConfig "../modules/bwa_index.config"

conf/subworkflows/fastq_align_dedup_bismark.config

@@ -10,20 +10,22 @@ includeConfig "../modules/bismark_summary.config"
 // Override SAMTOOLS_SORT for bismark workflow
 // In bismark, deduplication (BISMARK_DEDUPLICATE) happens BEFORE sorting,
 // so the BAM input to SAMTOOLS_SORT is already deduplicated when skip_deduplication is false
+// Note: RRBS also skips deduplication, so we check for both params.skip_deduplication and params.rrbs
 process {
-    withName: SAMTOOLS_SORT {
-        ext.prefix = params.skip_deduplication ? { "${meta.id}.sorted" } : { "${meta.id}.deduplicated.sorted" }
+    withName: 'NFCORE_METHYLSEQ:METHYLSEQ:FASTQ_ALIGN_DEDUP_BISMARK:SAMTOOLS_SORT' {
+        ext.prefix = { (params.skip_deduplication || params.rrbs) ? "${meta.id}.sorted" : "${meta.id}.deduplicated.sorted" }
         publishDir = [
             [
                 path: { "${params.outdir}/${params.aligner}/deduplicated/" },
                 mode: params.publish_dir_mode,
-                pattern: "*.deduplicated.sorted.bam"
+                pattern: "*.deduplicated.sorted.bam",
+                enabled: !(params.skip_deduplication || params.rrbs)
             ],
             [
                 path: { "${params.outdir}/${params.aligner}/alignments/" },
                 mode: params.publish_dir_mode,
                 pattern: "*.sorted.bam",
-                enabled: params.skip_deduplication
+                enabled: params.skip_deduplication || params.rrbs
             ]
         ]
     }

conf/subworkflows/fastq_align_dedup_bwamem.config

@@ -1,4 +1,50 @@
 includeConfig "../modules/bwamem_align.config"
 includeConfig "../modules/picard_addorreplacereadgroups.config"
 includeConfig "../modules/picard_markduplicates.config"
-includeConfig "../modules/samtools_index.config"
+includeConfig "../modules/samtools_stats.config"
+includeConfig "../modules/samtools_flagstat.config"
+includeConfig "../modules/samtools_idxstats.config"
+
+// Note: SAMTOOLS_INDEX is called twice in this workflow with different requirements:
+// 1. BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX - indexes alignment BAMs (before dedup)
+// 2. FASTQ_ALIGN_DEDUP_BWAMEM:SAMTOOLS_INDEX - indexes deduplicated BAMs (after markdup)
+// We need full process paths to distinguish them.
+
+process {
+    // SAMTOOLS_SORT in BAM_SORT_STATS_SAMTOOLS - publish to alignments when skip_deduplication
+    withName: 'NFCORE_METHYLSEQ:METHYLSEQ:FASTQ_ALIGN_DEDUP_BWAMEM:FASTQ_ALIGN_BWA:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
+        ext.prefix = { "${meta.id}.sorted" }
+        publishDir = [
+            [
+                path: { "${params.outdir}/${params.aligner}/alignments/" },
+                mode: params.publish_dir_mode,
+                pattern: "*.sorted.bam",
+                enabled: params.skip_deduplication
+            ]
+        ]
+    }
+
+    // SAMTOOLS_INDEX in BAM_SORT_STATS_SAMTOOLS - only publish when skip_deduplication
+    withName: 'NFCORE_METHYLSEQ:METHYLSEQ:FASTQ_ALIGN_DEDUP_BWAMEM:FASTQ_ALIGN_BWA:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_INDEX' {
+        publishDir = [
+            [
+                path: { "${params.outdir}/${params.aligner}/alignments/" },
+                mode: params.publish_dir_mode,
+                pattern: "*.bai",
+                enabled: params.skip_deduplication
+            ]
+        ]
+    }
+
+    // SAMTOOLS_INDEX after PICARD_MARKDUPLICATES - publish to deduplicated when deduplication runs
+    withName: 'NFCORE_METHYLSEQ:METHYLSEQ:FASTQ_ALIGN_DEDUP_BWAMEM:SAMTOOLS_INDEX' {
+        publishDir = [
+            [
+                path: { "${params.outdir}/${params.aligner}/deduplicated/" },
+                mode: params.publish_dir_mode,
+                pattern: "*.bai",
+                enabled: !params.skip_deduplication
+            ]
+        ]
+    }
+}

conf/subworkflows/fastq_align_dedup_bwameth.config

@@ -15,7 +15,7 @@ process {
                 path: { "${params.outdir}/${params.aligner}/alignments/" },
                 mode: params.publish_dir_mode,
                 pattern: "*.bam.bai",
-                enabled: params.save_align_intermeds
+                enabled: params.save_align_intermeds || params.skip_deduplication || params.rrbs
             ]
         ]
     }
@@ -26,7 +26,8 @@ process {
             [
                 path: { "${params.outdir}/${params.aligner}/deduplicated/" },
                 mode: params.publish_dir_mode,
-                pattern: "*.bam.bai"
+                pattern: "*.bam.bai",
+                enabled: !(params.skip_deduplication || params.rrbs)
             ]
         ]
     }

docs/output.md

@@ -91,7 +91,9 @@ bwameth/
 bwamem/
 ├── bwamem
 │   ├── alignments
+│   │   └── samtools_stats
 │   └── deduplicated
+│       └── picard_metrics
 ├── fastqc
 │   ├── Ecoli_10K_methylated_1_fastqc.html
 │   ├── Ecoli_10K_methylated_2_fastqc.html
@@ -107,15 +109,13 @@ bwamem/
 │   └── pipeline_dag_2024-12-13_05-36-34.html
 ├── rastair
 │   ├── call
-│   |   └── Ecoli_10K_methylated.markdup.sorted_CpG.rastair_call.tsv
+│   │   └── Ecoli_10K_methylated.rastair_call.txt
 │   ├── mbias
-│   |   └── Ecoli_10K_methylated.markdup.sorted_CpG.rastair_mbias.tsv
+│   │   └── Ecoli_10K_methylated.rastair_mbias.txt
 │   ├── mbiasparser
-│   │   ├── Ecoli_10K_methylated.markdup.sorted_CpG.rastair_mbias_processed.txt
-│   │   ├── Ecoli_10K_methylated.markdup.sorted_CpG.rastair_mbias_processed.csv
-│   │   └── Ecoli_10K_methylated.markdup.sorted_CpG.rastair_mbias_processed.pdf
-│   ├── methylkit
-|   │   └── Ecoli_10K_methylated.markdup.sorted_CpG.rastair_methylkit.txt.gz
+│   │   └── Ecoli_10K_methylated.rastair_mbias_processed.csv
+│   └── methylkit
+│       └── Ecoli_10K_methylated.rastair_methylkit.txt.gz
 └── trimgalore
     ├── fastqc
     └── logs
@@ -212,14 +212,18 @@ _Note that bismark can use either use Bowtie2 (default) or HISAT2 as alignment t
 
 **bwa-mem output directory: `results/bwamem/alignments/`**
 
-- `sample.bam`
-  - Aligned reads in BAM format.
-- `sample.bam.bai`
-  - Index of BAM file
+- `sample.sorted.bam`
+  - Sorted aligned reads in BAM format.
+  - **NB:** Only saved if `--save_align_intermeds` or `--skip_deduplication` is specified.
+- `sample.sorted.bam.bai`
+  - Index of sorted BAM file.
+  - **NB:** Only saved if `--save_align_intermeds` or `--skip_deduplication` is specified.
 - `samtools_stats/sample.flagstat`
   - Summary file describing the number of reads which aligned in different ways.
 - `samtools_stats/sample.stats`
   - Summary file giving lots of metrics about the aligned BAM file.
+- `samtools_stats/sample.idxstats`
+  - Summary file with alignment statistics per reference sequence.
 
 ### Deduplication
 
@@ -245,14 +249,10 @@ This step removes alignments with identical mapping position to avoid technical
 
 **bwa-mem output directory: `results/bwamem/deduplicated/`**
 
-- `sample.deduplicated.sorted.bam`
-  - Sorted BAM file with only unique alignments.
-- `sample.deduplicated.sorted.bam.bai`
-  - Index for sorted and deduped BAM file.
 - `sample.markdup.sorted.bam`
-  - BAM file with duplicated reads marked.
+  - Sorted BAM file with duplicated reads marked.
 - `sample.markdup.sorted.bam.bai`
-  - BAM file with duplicated reads marked.
+  - Index for the markdup BAM file.
 - `picard_metrics/sample.markdup.sorted.MarkDuplicates.metrics.txt`
   - Log file giving summary statistics about deduplication.
 
@@ -293,18 +293,14 @@ Filename abbreviations stand for the following reference alignment strands:
 
 **bwa-mem / TAPS workflow output directory: `results/rastair/`**
 
-- `call/samples.markdup.sorted_CpG.rastair_call.tsv`
+- `call/sample.rastair_call.txt`
   - Individual methylation calls file sorted by genomic coordinates and including cytosine context.
-- `mbias/sample.markdup.sorted_CpG.rastair_mbias.tsv`
+- `mbias/sample.rastair_mbias.txt`
   - Average conversion rate per position on the read per read pair (R1 and R2) original strand (OT, OB).
-- `mbiasparser/sample.markdup.sorted_CpG.rastair_mbias_processed.txt`
-  - File reporting the dynamic trimming to be passed to the rastair/call process
-- `mbiasparser/sample.markdup.sorted_CpG.rastair_mbias_processed.csv`
-  - File reporting the dynamic trimming to be passed to the rastair/call process
-- `mbiasparser/sample.markdup.sorted_CpG.rastair_mbias_processed.pdf`
-  - PDF file with the plot reporting the mbias and dynamic trimming selected for the sample
-- `methylkit/sample.markdup.sorted_CpG.rastair_methylkit.txt.gz`
-  - Individual methylation calls in a format digestible by MethytlKit R package.
+- `mbiasparser/sample.rastair_mbias_processed.csv`
+  - File reporting the dynamic trimming to be passed to the rastair/call process.
+- `methylkit/sample.rastair_methylkit.txt.gz`
+  - Individual methylation calls in a format digestible by MethylKit R package.
 
 ### Targeted Sequencing
 
@@ -333,9 +329,9 @@ BedGraph files are filtered using the BED file passed to `--target_regions_file`
 
 Bismark generates a HTML reports describing results for each sample, as well as a summary report for the whole run.
 
-**Output directory: `results/bismark_reports`**
+**Output directory: `results/bismark/reports/`**
 
-**Output directory: `results/bismark_summary`**
+**Output directory: `results/bismark/summary/`**
 
 ## Qualimap