Fix issue with optional workflow output (#6519)

Signed-off-by: Ben Sherman <bentshermann@gmail.com>
Co-authored-by: Jorge Ejarque <jorgee@users.noreply.github.com>

Author: bentsherman

modules/nextflow/src/main/groovy/nextflow/extension/DumpHelper.groovy

@@ -33,7 +33,7 @@ import org.yaml.snakeyaml.Yaml
 class DumpHelper {
 
     static def deepConvertToString(value, nullValue = '') {
-        if( value instanceof List )
+        if( value instanceof Collection )
             value.collect { it -> deepConvertToString(it) }
 
         else if( value instanceof Map )
@@ -53,7 +53,7 @@ class DumpHelper {
     }
 
     static String prettyPrint(value) {
-        if( value instanceof List || value instanceof Map || value instanceof Path )
+        if( value instanceof Collection || value instanceof Map || value instanceof Path )
             return prettyPrintJson(value)
         else
             return InvokerHelper.inspect(value)

modules/nextflow/src/main/groovy/nextflow/extension/PublishOp.groovy

@@ -141,7 +141,7 @@ class PublishOp {
         // the resulting mapping to create a saveAs closure
         final mapping = dsl.build()
         if( mapping instanceof Map<String,String> )
-            return { filename -> outputDir.resolve(mapping[filename]) }
+            return { filename -> filename in mapping ? outputDir.resolve(mapping[filename]) : null }
 
         // if the resolved publish path is a string, resolve it
         // against the base output directory
@@ -172,6 +172,8 @@ class PublishOp {
         }
 
         private void publish0(Path source, String target) {
+            if( source == null || target == null )
+                return
             log.trace "Publishing ${source} to ${target}"
             if( mapping == null )
                 mapping = [:]
@@ -278,11 +280,11 @@ class PublishOp {
         if( value instanceof Map ) {
             return value.collectEntries { k, v ->
                 if( v instanceof Path )
-                    return Map.entry(k, normalizePath(v, targetResolver))
+                    return [k, normalizePath(v, targetResolver)]
                 if( v instanceof Collection<Path> )
-                    return Map.entry(k, normalizePaths(v, targetResolver))
+                    return [k, normalizePaths(v, targetResolver)]
                 if( v instanceof Map )
-                    return Map.entry(k, normalizePaths(v, targetResolver))
+                    return [k, normalizePaths(v, targetResolver)]
                 return [k, v]
             }
         }
@@ -307,8 +309,11 @@ class PublishOp {
         // if the target resolver is a closure, use it to transform
         // the source filename to the target path
         if( targetResolver instanceof Closure<Path> ) {
+            // note: the closure can return null to e.g. not
+            // publish specific files
             final relPath = sourceDir.relativize(path).toString()
-            return (targetResolver.call(relPath) as Path).normalize()
+            final resolvedPath = targetResolver.call(relPath) as Path
+            return resolvedPath?.normalize()
         }
 
         // if the target resolver is a directory, resolve the source

tests/checks/output-dsl.nf/.checks

@@ -1,7 +1,5 @@
-#
-# run normal mode
-#
-echo First run
+
+echo "Initial run"
 $NXF_RUN --save_bam_bai | tee stdout
 
 [[ $(grep INFO .nextflow.log | grep -c 'Submitted process > fastqc') == 3 ]] || false
@@ -26,10 +24,7 @@ $NXF_RUN --save_bam_bai | tee stdout
 [[ -f results/summary_data/fastqc.txt ]] || false
 
 
-#
-# one more time to make sure 'overwrite' is fine
-#
-echo Second run
+echo "Run again with overwrite enabled"
 $NXF_RUN --save_bam_bai | tee stdout
 
 [[ $(grep INFO .nextflow.log | grep -c 'Submitted process > fastqc') == 3 ]] || false
@@ -54,10 +49,29 @@ $NXF_RUN --save_bam_bai | tee stdout
 [[ -f results/summary_data/fastqc.txt ]] || false
 
 
-#
-# clean & run resume mode
-#
-echo Third run
+echo "Run with some outputs disabled"
+rm -rf results
+
+$NXF_RUN -resume | tee stdout
+
+[[ $(grep INFO .nextflow.log | grep -c 'Cached process > fastqc') == 3 ]] || false
+[[ $(grep INFO .nextflow.log | grep -c 'Cached process > align') == 3 ]] || false
+[[ $(grep INFO .nextflow.log | grep -c 'Cached process > quant') == 3 ]] || false
+
+[[ -f results/log/alpha.fastqc.log ]] || false
+[[ -f results/log/beta.fastqc.log ]] || false
+[[ -f results/log/delta.fastqc.log ]] || false
+[[ ! -e results/align ]] || false
+[[ -L results/quant/alpha ]] || false
+[[ -L results/quant/beta ]] || false
+[[ -L results/quant/delta ]] || false
+[[ -f results/samples.csv ]] || false
+[[ -f results/summary_report.html ]] || false
+[[ -f results/summary_data/data.json ]] || false
+[[ -f results/summary_data/fastqc.txt ]] || false
+
+
+echo "Resumed run"
 rm -rf results
 
 $NXF_RUN --save_bam_bai -resume | tee stdout

tests/output-dsl.nf

@@ -93,8 +93,8 @@ workflow {
       [
         id: id,
         fastqc: fastqc,
-        bam: params.save_bam_bai ? bam : null,
-        bai: params.save_bam_bai ? bai : null,
+        bam: bam,
+        bai: bai,
         quant: quant
       ]
     }
@@ -114,8 +114,8 @@ output {
   samples {
     path { sample ->
       sample.fastqc >> 'log/'
-      sample.bam >> 'align/'
-      sample.bai >> 'align/'
+      sample.bam >> (params.save_bam_bai ? 'align/' : null)
+      sample.bai >> (params.save_bam_bai ? 'align/' : null)
       sample.quant >> "quant/${sample.id}"
     }
     index {