Merge pull request #121 from nextflow-io/2.4.0dev

Release 2.4.0

Author: nvnieuwk

.github/workflows/build-docs.yml

@@ -19,7 +19,7 @@ jobs:
           python-version: 3.x
       - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
       - name: Obtain version from MANIFEST.MF
-        run: echo "plugin_version=$(grep "Plugin-Version" plugins/nf-schema/src/resources/META-INF/MANIFEST.MF | awk '{print $2}' | awk -F '.' '{print $1"."$2}')" >> $GITHUB_ENV
+        run: echo "plugin_version=$(grep "version = " build.gradle | sed -e 's/version = //' | cut -d "'" -f 2)" >> $GITHUB_ENV
       - name: Setup git user
         run: |
           git config --global user.name "${{github.actor}}"

.github/workflows/build.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java_version: [11, 17]
+        java_version: [11, 17, 21]
 
     steps:
       - name: Environment

CHANGELOG.md

@@ -1,5 +1,31 @@
 # nextflow-io/nf-schema: Changelog
 
+# Version 2.4.0
+
+## New features
+
+1. Added a new configuration option: `validation.maxErrValSize` which sets the maximum length that a value in an error message can be. The default is set to 150 characters.
+2. Added a new function: `validate()` that can be used to validate any data structure using a JSON schema.
+
+## Bug fixes
+
+1. Move the unpinned version check to an observer. This makes sure the warning is always shown and not only when importing a function.
+2. Added a missing inherited method to the observer to fix issues with workflow output publishing
+3. Fixed unexpected failures with samplesheet schemas using `anyOf`, `allOf` and `oneOf`
+4. Fixed an error with help messages when the `type` keyword was missing
+5. Fix compilation errors in Java 21
+
+## Improvements
+
+1. Slow uniqueness check (> 2hrs for 100k samples) made 400x faster by switching from `findAll` to a `subMap` for isolating the required unique fields.
+2. `patternProperties` now has greater support, with no warnings about invalid parameters which actually match a pattern
+3. Added better error handling and debug messages to the configuration parser.
+
+## Changes
+
+1. Refactored the whole codebase to make future development easier
+2. Bumped the minimal Nextflow version to `24.10.0`
+
 # Version 2.3.0 - Hakodate
 
 ## Bug fixes

Makefile

@@ -1,65 +1,23 @@
-
-config ?= compileClasspath
-version ?= $(shell grep 'Plugin-Version' plugins/nf-schema/src/resources/META-INF/MANIFEST.MF | awk '{ print $$2 }')
-
-ifdef module 
-mm = :${module}:
-else 
-mm = 
-endif 
+# Build the plugin
+assemble:
+	./gradlew assemble
 
 clean:
+	rm -rf .nextflow*
+	rm -rf work
+	rm -rf build
 	./gradlew clean
 
-compile:
-	./gradlew compileGroovy
-	@echo "DONE `date`"
-
-
-check:
-	./gradlew check
-
-
-#
-# Show dependencies try `make deps config=runtime`, `make deps config=google`
-#
-deps:
-	./gradlew -q ${mm}dependencies --configuration ${config}
-
-deps-all:
-	./gradlew -q dependencyInsight --configuration ${config} --dependency ${module}
-
-#
-# Refresh SNAPSHOTs dependencies
-#
-refresh:
-	./gradlew --refresh-dependencies 
-
 #
 # Run all tests or selected ones
 #
 test:
-ifndef class
-	./gradlew ${mm}test
-else
-	./gradlew ${mm}test --tests ${class}
-endif
-
+	./gradlew test
 
+# Install the plugin into local nextflow plugins dir
 install:
-	./gradlew copyPluginZip
-	rm -rf ${HOME}/.nextflow/plugins/nf-schema-${version}
-	cp -r build/plugins/nf-schema-${version} ${HOME}/.nextflow/plugins/nf-schema-${version}
-
-#
-# Upload JAR artifacts to Maven Central
-#
-upload:
-	./gradlew upload
-
-
-upload-plugins:
-	./gradlew plugins:upload
+	./gradlew install
 
-publish-index:
-	./gradlew plugins:publishIndex
+# Publish the plugin
+release:
+	./gradlew releasePlugin

README.md

@@ -25,7 +25,7 @@ Declare the plugin in your Nextflow pipeline configuration file:
 
 ```groovy title="nextflow.config"
 plugins {
-  id 'nf-schema@2.2.0'
+  id 'nf-schema@2.4.0'
 }
 ```
 
@@ -65,7 +65,7 @@ validation {
 
 - Java 11 or later
 - <https://github.com/harrel56/json-schema>
-- Nextflow 23.10.0 or later
+- Nextflow 24.10.0 or later
 
 ## Slack channel
 

build.gradle

@@ -0,0 +1,33 @@
+plugins {
+    id 'io.nextflow.nextflow-plugin' version '0.0.1-alpha2'
+}
+
+dependencies {
+    implementation 'org.json:json:20240303'
+    implementation 'dev.harrel:json-schema:1.5.0'
+    implementation 'com.sanctionco.jmail:jmail:1.6.3' // Needed for e-mail format validation
+}
+
+version = '2.4.0'
+
+nextflowPlugin {
+    nextflowVersion = '24.10.0'
+
+    provider = 'nextflow'
+    className = 'nextflow.validation.ValidationPlugin'
+    extensionPoints = [
+        'nextflow.validation.ValidationExtension',
+        'nextflow.validation.ValidationObserverFactory'
+    ]
+
+    publishing {
+        github {
+            repository = 'nextflow-io/nf-schema'
+            userName = project.findProperty('github_username')
+            authToken = project.findProperty('github_access_token')
+            email = project.findProperty('github_commit_email')
+
+            indexUrl = 'https://github.com/nextflow-io/plugins/blob/main/plugins.json'
+        }
+    }
+}

buildSrc/src/main/groovy/io.nextflow.groovy-common-conventions.gradle

@@ -14,7 +14,7 @@ repositories {
 
 java {
     toolchain {
-        languageVersion = JavaLanguageVersion.of(19)
+        languageVersion = JavaLanguageVersion.of(21)
     }
 }
 

docs/configuration/configuration.md

@@ -95,6 +95,22 @@ This option does exactly the same as `validation.ignoreParams`, but provides pip
 validation.defaultIgnoreParams = ["param1", "param2"] // default: []
 ```
 
+## maxErrValSize
+
+Configure the maximum characters of a value that may be shown in an error message. It takes a whole number above or equal to `1`. A value will be truncated when it goes over the maximum amount of characters.
+
+Setting this option to `-1` will allow any amount of characters for values.
+
+See the below example where the limit is to 20 characters:
+
+```
+* --test (abcdefghij...qrstuvwxyz): Value is [string] but should be [integer]
+```
+
+```groovy
+validation.maxErrValSize = 100 // default: 150
+```
+
 ## help
 
 The `validation.help` config scope can be used to configure the creation of the help message.

docs/contributing/setup.md

@@ -5,15 +5,15 @@ description: How to contribute to nf-schema
 
 # Getting started with plugin development
 
-## Compiling
+## Tests
 
-To compile and run the tests use the following command:
+To run the tests use the following command:
 
 ```bash
-./gradlew check
+make test
 ```
 
-## Launch it with installed Nextflow
+## Install and use in pipelines
 
 !!! warning
 
@@ -39,33 +39,6 @@ plugins {
 }
 ```
 
-## Launch it with a local version of Nextflow
-
-- Clone the Nextflow repo into a sibling directory
-
-```bash
-cd .. && git clone https://github.com/nextflow-io/nextflow
-cd nextflow && ./gradlew exportClasspath
-```
-
-- Append to the `settings.gradle` in this project the following line:
-
-```bash
-includeBuild('../nextflow')
-```
-
-- Compile the plugin code
-
-```bash
-./gradlew compileGroovy
-```
-
-- Run nextflow with this command:
-
-```bash
-./launch.sh run -plugins nf-schema <script/pipeline name> [pipeline params]
-```
-
 ## Change and preview the docs
 
 The docs are generated using [Material for MkDocs](https://squidfunk.github.io/mkdocs-material/).
@@ -82,3 +55,20 @@ mkdocs serve
 ```
 
 To preview the docs, open the URL provided by mkdocs in your browser.
+
+## Release and publish the plugin
+
+1. In `build.gradle` make sure that:
+   - `version` matches the desired release version,
+   - `github.repository` matches the repository of the plugin,
+   - `github.indexUrl` points to your fork of the plugins index repository.
+2. Create a file named `$HOME/.gradle/gradle.properties`, where `$HOME` is your home directory. Add the following properties:
+   - `github_username`: The GitHub username granting access to the plugin repository.
+   - `github_access_token`: The GitHub access token required to upload and commit changes to the plugin repository.
+   - `github_commit_email`: The email address associated with your GitHub account.
+3. Update the [changelog](./CHANGELOG.md).
+4. Build and publish the plugin to your GitHub repository:
+   ```bash
+   make release
+   ```
+5. Create a pull request against the [nextflow-io/plugins](https://github.com/nextflow-io/plugins/blob/main/plugins.json) repository to make the plugin publicly accessible.

docs/validate/validate.md

@@ -0,0 +1,32 @@
+---
+title: Validate common data structures
+description: Function to validate common data structures
+---
+
+# Validate common data structures
+
+The `validate` function can be used to validate common data structures.
+
+```groovy
+include { validate } from 'plugin/nf-schema'
+
+validate(<input>, <schema>)
+```
+
+This function takes two positional arguments:
+
+1. The data that needs to be validated
+2. The path to the schema used to validate the input data
+
+The tested data structures are listed below, however more data structures might by supported too:
+
+- Maps
+- Lists
+- String
+- Integers
+- Booleans
+
+`validate` also has the following optional arguments:
+
+- `exitOnError`: Exit the pipeline on validation failure and show the error message. The function will output the errors when this option is set to `false` (default: `true`)
+  - `validate(<input>, <schema>, exitOnError:true|false)`