Addressable data store (aka CID store)

build.gradle

@@ -237,7 +237,7 @@ task compile {
 
 def getRuntimeConfigs() {
     def names = subprojects
-            .findAll { prj -> prj.name in ['nextflow','nf-commons','nf-httpfs','nf-lang'] }
+            .findAll { prj -> prj.name in ['nextflow','nf-cid','nf-commons','nf-httpfs','nf-lang'] }
             .collect { it.name }
 
     FileCollection result = null
@@ -263,7 +263,7 @@ task exportClasspath {
         def home = System.getProperty('user.home')
         def all = getRuntimeConfigs()
         def libs = all.collect { File file ->  /*println file.canonicalPath.replace(home, '$HOME');*/ file.canonicalPath;  }
-        ['nextflow','nf-commons','nf-httpfs','nf-lang'].each {libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath }
+        ['nextflow','nf-cid','nf-commons','nf-httpfs','nf-lang'].each {libs << file("modules/$it/build/libs/${it}-${version}.jar").canonicalPath }
         file('.launch.classpath').text = libs.unique().join(':')
     }
 }
@@ -276,7 +276,7 @@ ext.nexusEmail = project.findProperty('nexusEmail')
 // `signing.keyId` property needs to be defined in the `gradle.properties` file
 ext.enableSignArchives = project.findProperty('signing.keyId')
 
-ext.coreProjects = projects( ':nextflow', ':nf-commons', ':nf-httpfs', ':nf-lang' )
+ext.coreProjects = projects( ':nextflow', ':nf-cid', ':nf-commons', ':nf-httpfs', ':nf-lang' )
 
 configure(coreProjects) {
     group = 'io.nextflow'

modules/nextflow/build.gradle

@@ -53,7 +53,7 @@ dependencies {
     api 'io.seqera:lib-trace:0.1.0'
 
     testImplementation 'org.subethamail:subethasmtp:3.1.7'
-
+    testImplementation (project(':nf-cid'))
     // test configuration
     testFixturesApi ("org.apache.groovy:groovy-test:4.0.26") { exclude group: 'org.apache.groovy' }
     testFixturesApi ("org.objenesis:objenesis:3.4")

modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy

@@ -0,0 +1,231 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.cli
+
+import java.nio.file.Paths
+
+import com.beust.jcommander.Parameter
+import com.beust.jcommander.Parameters
+import groovy.transform.CompileStatic
+import nextflow.config.ConfigBuilder
+import nextflow.config.ConfigMap
+import nextflow.exception.AbortOperationException
+import nextflow.plugin.Plugins
+import org.pf4j.ExtensionPoint
+
+/**
+ * CID command line interface
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+@CompileStatic
+@Parameters(commandDescription = "Explore workflows CID metadata")
+class CmdCid extends CmdBase implements UsageAware {
+
+    private static final String NAME = 'cid'
+
+    interface CidCommand extends ExtensionPoint {
+        void log(ConfigMap config)
+        void show(ConfigMap config, List<String> args)
+        void lineage(ConfigMap config, List<String> args)
+    }
+
+    interface SubCmd {
+        String getName()
+        String getDescription()
+        void apply(List<String> args)
+        void usage()
+    }
+
+    private List<SubCmd> commands = new ArrayList<>()
+
+    private CidCommand operation
+
+    private ConfigMap config
+
+    CmdCid() {
+        commands << new CmdLog()
+        commands << new CmdShow()
+        commands << new CmdLineage()
+    }
+
+    @Parameter(hidden = true)
+    List<String> args
+
+    @Override
+    String getName() {
+        return NAME
+    }
+
+    @Override
+    void run() {
+        if( !args ) {
+            return
+        }
+        // setup the plugins system and load the secrets provider
+        Plugins.init()
+        // load the config
+        this.config = new ConfigBuilder()
+            .setOptions(launcher.options)
+            .setBaseDir(Paths.get('.'))
+            .build()
+        // init plugins
+        Plugins.load(config)
+        // load the command operations
+        this.operation = Plugins.getExtension(CidCommand)
+        if( !operation )
+            throw new IllegalStateException("Unable to load CID plugin")
+        // consume the first argument
+        getCmd(args).apply(args.drop(1))
+    }
+
+    /**
+     * Print the command usage help
+     */
+    void usage() {
+        usage(args)
+    }
+
+    /**
+     * Print the command usage help
+     *
+     * @param args The arguments as entered by the user
+     */
+    void usage(List<String> args) {
+        if( !args ) {
+            List<String> result = []
+            result << this.getClass().getAnnotation(Parameters).commandDescription()
+            result << "Usage: nextflow $NAME <sub-command> [options]".toString()
+            result << ''
+            result << 'Commands:'
+            int len = 0
+            commands.forEach {len = it.name.size() > len ? it.name.size() : len }
+            commands.sort(){it.name}.each { result << "  ${it.name.padRight(len)}\t${it.description}".toString()  }
+            result << ''
+            println result.join('\n').toString()
+        }
+        else {
+            def sub = commands.find { it.name == args[0] }
+            if( sub )
+                sub.usage()
+            else {
+                throw new AbortOperationException("Unknown $NAME sub-command: ${args[0]}")
+            }
+        }
+    }
+
+    protected SubCmd getCmd(List<String> args) {
+
+        def cmd = commands.find { it.name == args[0] }
+        if( cmd ) {
+            return cmd
+        }
+
+        def matches = commands.collect{ it.name }.closest(args[0])
+        def msg = "Unknown cloud sub-command: ${args[0]}"
+        if( matches )
+            msg += " -- Did you mean one of these?\n" + matches.collect { "  $it"}.join('\n')
+        throw new AbortOperationException(msg)
+    }
+
+    class CmdLog implements SubCmd {
+
+        @Override
+        String getName() {
+            return 'log'
+        }
+
+        @Override
+        String getDescription() {
+            return 'Print the CID execution log'
+        }
+
+        @Override
+        void apply(List<String> args) {
+            if (args.size() != 0) {
+                println("ERROR: Incorrect number of parameters")
+                usage()
+                return
+            }
+            operation.log(config)
+        }
+
+        @Override
+        void usage() {
+            println description
+            println "Usage: nextflow $NAME $name"
+        }
+    }
+
+    class CmdShow implements SubCmd{
+
+        @Override
+        String getName() {
+            return 'show'
+        }
+
+        @Override
+        String getDescription() {
+            return 'Print the description of a CID reference'
+        }
+
+        void apply(List<String> args) {
+            if (args.size() != 1) {
+                println("ERROR: Incorrect number of parameters")
+                usage()
+                return
+            }
+
+            operation.show(config, args)
+        }
+
+        @Override
+        void usage() {
+            println description
+            println "Usage: nextflow $NAME $name <CID reference> "
+        }
+    }
+
+    class CmdLineage implements SubCmd {
+
+        @Override
+        String getName() { 'lineage' }
+
+        @Override
+        String getDescription() {
+            return 'Render a lineage graph for a workflow output'
+        }
+
+        void apply(List<String> args) {
+            if (args.size() != 2) {
+                println("ERROR: Incorrect number of parameters")
+                usage()
+                return
+            }
+
+            operation.lineage(config, args)
+        }
+
+        @Override
+        void usage() {
+            println description
+            println "Usage: nextflow $NAME $name <workflow output CID> <html output file>"
+        }
+
+    }
+}

modules/nextflow/src/main/groovy/nextflow/cli/Launcher.groovy

@@ -106,7 +106,8 @@ class Launcher {
                 new CmdHelp(),
                 new CmdSelfUpdate(),
                 new CmdPlugin(),
-                new CmdInspect()
+                new CmdInspect(),
+                new CmdCid()
         ]
 
         if(SecretsLoader.isEnabled())

modules/nextflow/src/main/groovy/nextflow/dag/MermaidHtmlRenderer.groovy

@@ -33,7 +33,7 @@ class MermaidHtmlRenderer implements DagRenderer {
         file.text = template.replace('REPLACE_WITH_NETWORK_DATA', network)
     }
 
-    private String readTemplate() {
+    static String readTemplate() {
         final writer = new StringWriter()
         final res = MermaidHtmlRenderer.class.getResourceAsStream('mermaid.dag.template.html')
         int ch

modules/nextflow/src/main/groovy/nextflow/file/RealPathAware.groovy

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2013-2025, Seqera Labs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package nextflow.file
+
+import java.nio.file.LinkOption
+import java.nio.file.Path
+
+/**
+ * Marker interface for objects that represent a real path.
+ *
+ * This interface is used in the { @link nextflow.processor.TaskProcessor } when managing the foreign file staging.
+ *
+ * @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
+ */
+interface RealPathAware {
+    /**
+     * Returns the real path
+     * @param options
+     * @return
+     */
+    Path toRealPath(LinkOption... options)
+}

modules/nextflow/src/main/groovy/nextflow/processor/TaskId.groovy

@@ -38,6 +38,8 @@ class TaskId extends Number implements Comparable, Serializable, Cloneable {
 
     private final int value
 
+    int getValue() { value }
+
     static TaskId of( value ) {
         if( value instanceof Integer )
             return new TaskId(value)

modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy

@@ -15,8 +15,6 @@
  */
 package nextflow.processor
 
-import nextflow.trace.TraceRecord
-
 import static nextflow.processor.ErrorStrategy.*
 
 import java.lang.reflect.InvocationTargetException
@@ -80,6 +78,7 @@ import nextflow.file.FileHelper
 import nextflow.file.FileHolder
 import nextflow.file.FilePatternSplitter
 import nextflow.file.FilePorter
+import nextflow.file.RealPathAware
 import nextflow.plugin.Plugins
 import nextflow.processor.tip.TaskTipProvider
 import nextflow.script.BaseScript
@@ -106,6 +105,7 @@ import nextflow.script.params.TupleInParam
 import nextflow.script.params.TupleOutParam
 import nextflow.script.params.ValueInParam
 import nextflow.script.params.ValueOutParam
+import nextflow.trace.TraceRecord
 import nextflow.util.ArrayBag
 import nextflow.util.BlankSeparatedList
 import nextflow.util.CacheHelper
@@ -1939,6 +1939,9 @@ class TaskProcessor {
 
             if( item instanceof Path || coerceToPath ) {
                 def path = normalizeToPath(item)
+                if (path instanceof RealPathAware){
+                    path = path.toRealPath()
+                }
                 def target = executor.isForeignFile(path) ? batch.addToForeign(path) : path
                 def holder = new FileHolder(target)
                 files << holder
@@ -2274,7 +2277,7 @@ class TaskProcessor {
      * @return The list of paths of scripts in the project bin folder referenced in the task command
      */
     @Memoized
-    protected List<Path> getTaskBinEntries(String script) {
+    public List<Path> getTaskBinEntries(String script) {
         List<Path> result = []
         def tokenizer = new StringTokenizer(script," \t\n\r\f()[]{};&|<>`")
         while( tokenizer.hasMoreTokens() ) {
@@ -2307,7 +2310,7 @@ class TaskProcessor {
         log.info(buffer.toString())
     }
 
-    protected Map<String,Object> getTaskGlobalVars(TaskRun task) {
+    public Map<String,Object> getTaskGlobalVars(TaskRun task) {
         final result = task.getGlobalVars(ownerScript.binding)
         final directives = getTaskExtensionDirectiveVars(task)
         result.putAll(directives)

modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy

@@ -990,5 +990,9 @@ class TaskRun implements Cloneable {
     CondaConfig getCondaConfig() {
         return processor.session.getCondaConfig()
     }
+
+    String getStubSource(){
+        return config?.getStubBlock()?.source
+    }
 }