Skip format validation for cloud storage paths (S3, Azure, GCS)

CHANGELOG.md

@@ -1,5 +1,11 @@
 # nextflow-io/nf-schema: Changelog
 
+# Version 2.6.2 (Unreleased)
+
+## Bug fixes
+
+1. Fixed a bug where cloud storage paths (S3, Azure, Google Cloud Storage) would fail validation with "could not validate file format" errors when the paths could not be accessed due to missing credentials or permissions. Cloud storage paths now gracefully skip validation when inaccessible, while still validating when access is available.
+
 # Version 2.6.1
 
 This is a quick patch release to bump the Nextflow Gradle plugin to `1.0.0-beta.11`. This will enable support for the configuration options in the Nextflow language server.

src/main/groovy/nextflow/validation/utils/Common.groovy

@@ -19,6 +19,15 @@ import java.util.stream.IntStream
 @Slf4j
 public class Common {
 
+    private static final List<String> CLOUD_STORAGE_SCHEMES = ['s3://', 'az://', 'gs://']
+
+    //
+    // Check if a path string is a cloud storage path (S3, Azure, GCS)
+    //
+    public static boolean isCloudStoragePath(String value) {
+        return CLOUD_STORAGE_SCHEMES.any { value.startsWith(it) }
+    }
+
     //
     // Get full path based on the base directory of the pipeline run
     //

src/main/groovy/nextflow/validation/validators/evaluators/ExistsEvaluator.groovy

@@ -8,6 +8,8 @@ import nextflow.Nextflow
 import groovy.util.logging.Slf4j
 import java.nio.file.Path
 
+import static nextflow.validation.utils.Common.isCloudStoragePath
+
 /**
  * @author : nvnieuwk <nicolas.vannieuwkerke@ugent.be>
  */
@@ -46,6 +48,12 @@ class ExistsEvaluator implements Evaluator {
             }
 
         } catch (Exception e) {
+            // For cloud storage paths, skip validation gracefully if we can't access them
+            // (e.g., due to missing credentials or permissions)
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping existence validation for cloud storage path '${value}' due to exception: ${e.message}")
+                return Evaluator.Result.success()
+            }
             return Evaluator.Result.failure("could not check existence of '${value}': ${e.message}" as String)
         }
 

src/main/groovy/nextflow/validation/validators/evaluators/FormatDirectoryPathEvaluator.groovy

@@ -8,6 +8,8 @@ import nextflow.Nextflow
 import groovy.util.logging.Slf4j
 import java.nio.file.Path
 
+import static nextflow.validation.utils.Common.isCloudStoragePath
+
 /**
  * @author : nvnieuwk <nicolas.vannieuwkerke@ugent.be>
  */
@@ -32,7 +34,13 @@ class FormatDirectoryPathEvaluator implements Evaluator {
                 file.exists() // Do an exists check to see if the file can be correctly accessed
             }
         } catch (Exception e) {
-            return Evaluator.Result.failure("could not validate file format of '${value}': ${e.message}" as String)
+            // For cloud storage paths, skip validation gracefully if we can't access them
+            // (e.g., due to missing credentials or permissions)
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping validation for cloud storage path '${value}' due to exception: ${e.message}")
+                return Evaluator.Result.success()
+            }
+            return Evaluator.Result.failure("could not validate directory format of '${value}': ${e.message}" as String)
         }
 
         // Actual validation logic

src/main/groovy/nextflow/validation/validators/evaluators/FormatFilePathEvaluator.groovy

@@ -8,6 +8,8 @@ import nextflow.Nextflow
 import groovy.util.logging.Slf4j
 import java.nio.file.Path
 
+import static nextflow.validation.utils.Common.isCloudStoragePath
+
 /**
  * @author : nvnieuwk <nicolas.vannieuwkerke@ugent.be>
  */
@@ -32,6 +34,12 @@ class FormatFilePathEvaluator implements Evaluator {
                 file.exists() // Do an exists check to see if the file can be correctly accessed
             }
         } catch (Exception e) {
+            // For cloud storage paths, skip validation gracefully if we can't access them
+            // (e.g., due to missing credentials or permissions)
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping validation for cloud storage path '${value}' due to exception: ${e.message}")
+                return Evaluator.Result.success()
+            }
             return Evaluator.Result.failure("could not validate file format of '${value}': ${e.message}" as String)
         }
 

src/main/groovy/nextflow/validation/validators/evaluators/FormatFilePathPatternEvaluator.groovy

@@ -8,14 +8,16 @@ import nextflow.Nextflow
 import groovy.util.logging.Slf4j
 import java.nio.file.Path
 
+import static nextflow.validation.utils.Common.isCloudStoragePath
+
 /**
  * @author : nvnieuwk <nicolas.vannieuwkerke@ugent.be>
  */
 
 @Slf4j
 class FormatFilePathPatternEvaluator implements Evaluator {
     // The string should be a path pattern
-  
+
     @Override
     public Evaluator.Result evaluate(EvaluationContext ctx, JsonNode node) {
         // To stay consistent with other keywords, types not applicable to this keyword should succeed
@@ -32,12 +34,23 @@ class FormatFilePathPatternEvaluator implements Evaluator {
                 file.exists() // Do an exists check to see if the file can be correctly accessed
             }
         } catch (Exception e) {
+            // For cloud storage paths, skip validation gracefully if we can't access them
+            // (e.g., due to missing credentials or permissions)
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping validation for cloud storage path '${value}' due to exception: ${e.message}")
+                return Evaluator.Result.success()
+            }
             return Evaluator.Result.failure("could not validate file format of '${value}': ${e.message}" as String)
         }
         // Actual validation logic
         def List<String> errors = []
 
         if(files.size() == 0) {
+            // For cloud storage paths, empty results may just mean we can't list the bucket
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping validation for cloud storage glob pattern '${value}': no files found (may be due to permissions)")
+                return Evaluator.Result.success()
+            }
             return Evaluator.Result.failure("No files were found using the glob pattern '${value}'" as String)
         }
         files.each { file ->

src/main/groovy/nextflow/validation/validators/evaluators/SchemaEvaluator.groovy

@@ -10,6 +10,7 @@ import groovy.util.logging.Slf4j
 import java.nio.file.Path
 
 import static nextflow.validation.utils.Common.getBasePath
+import static nextflow.validation.utils.Common.isCloudStoragePath
 import static nextflow.validation.utils.Files.fileToJson
 import nextflow.validation.config.ValidationConfig
 import nextflow.validation.validators.JsonSchemaValidator
@@ -43,11 +44,31 @@ class SchemaEvaluator implements Evaluator {
         def String value = node.asString()
 
         // Actual validation logic
-        def Path file = Nextflow.file(value)
+        def Path file
+        try {
+            file = Nextflow.file(value)
+        } catch (Exception e) {
+            // For cloud storage paths, skip validation gracefully if we can't access them
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping schema validation for cloud storage path '${value}' due to exception: ${e.message}")
+                return Evaluator.Result.success()
+            }
+            throw e
+        }
+
         // Don't validate if the file does not exist or is a directory
-        if(!file.exists() || file.isDirectory()) {
-            log.debug("Could not validate the file ${file.toString()}")
-            return Evaluator.Result.success()
+        try {
+            if(!file.exists() || file.isDirectory()) {
+                log.debug("Could not validate the file ${file.toString()}")
+                return Evaluator.Result.success()
+            }
+        } catch (Exception e) {
+            // For cloud storage paths, skip validation gracefully if we can't check existence
+            if (isCloudStoragePath(value)) {
+                log.debug("Skipping schema validation for cloud storage path '${value}' due to exception: ${e.message}")
+                return Evaluator.Result.success()
+            }
+            throw e
         }
 
         log.debug("Started validating ${file.toString()}")

src/test/groovy/nextflow/validation/ValidateParametersTest.groovy

@@ -1397,4 +1397,76 @@ class ValidateParametersTest extends Dsl2Spec{
         noExceptionThrown()
         stdout == ["* --testing: test", "* --genomebutlonger: true"]
     }
+
+    def 'should validate S3 storage paths without errors' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_cloud_path.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.s3_file = 's3://my-bucket/path/to/file.txt'
+            params.s3_directory = 's3://my-bucket/path/to/directory/'
+            include { validateParameters } from 'plugin/nf-schema'
+
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'should validate Google Cloud Storage paths without errors' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_cloud_path.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.gs_file = 'gs://my-bucket/path/to/file.txt'
+            params.gs_directory = 'gs://my-bucket/path/to/directory/'
+            include { validateParameters } from 'plugin/nf-schema'
+
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
+
+    def 'should validate Azure storage paths without errors' () {
+        given:
+        def schema = Path.of('src/testResources/nextflow_schema_cloud_path.json').toAbsolutePath().toString()
+        def SCRIPT = """
+            params.az_file = 'az://my-container/path/to/file.txt'
+            params.az_directory = 'az://my-container/path/to/directory/'
+            include { validateParameters } from 'plugin/nf-schema'
+
+            validateParameters(parameters_schema: '$schema')
+        """
+
+        when:
+        def config = [:]
+        def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
+        def stdout = capture
+                .toString()
+                .readLines()
+                .findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
+
+        then:
+        noExceptionThrown()
+        !stdout
+    }
 }

src/testResources/nextflow_schema_cloud_path.json

@@ -0,0 +1,39 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://raw.githubusercontent.com/nf-core/nf-schema/master/examples/cloudpath/nextflow_schema.json",
+    "title": "Cloud storage path validation test",
+    "description": "Test schema for cloud storage path validation",
+    "type": "object",
+    "properties": {
+        "s3_file": {
+            "type": "string",
+            "format": "file-path",
+            "description": "S3 storage file path"
+        },
+        "s3_directory": {
+            "type": "string",
+            "format": "directory-path",
+            "description": "S3 storage directory path"
+        },
+        "gs_file": {
+            "type": "string",
+            "format": "file-path",
+            "description": "Google Cloud Storage file path"
+        },
+        "gs_directory": {
+            "type": "string",
+            "format": "directory-path",
+            "description": "Google Cloud Storage directory path"
+        },
+        "az_file": {
+            "type": "string",
+            "format": "file-path",
+            "description": "Azure storage file path"
+        },
+        "az_directory": {
+            "type": "string",
+            "format": "directory-path",
+            "description": "Azure storage directory path"
+        }
+    }
+}