Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add CID H2 plugin #5889

Merged
merged 5 commits into from
Mar 20, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 12 additions & 7 deletions modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,10 @@ import nextflow.config.ConfigBuilder
import nextflow.config.ConfigMap
import nextflow.exception.AbortOperationException
import nextflow.plugin.Plugins
import org.pf4j.ExtensionPoint

/**
* CID command line interface
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
Expand All @@ -36,7 +39,7 @@ class CmdCid extends CmdBase implements UsageAware {

private static final String NAME = 'cid'

interface CidOperation {
interface CidCommand extends ExtensionPoint {
void log(ConfigMap config)
void show(ConfigMap config, List<String> args)
void lineage(ConfigMap config, List<String> args)
Expand All @@ -51,7 +54,7 @@ class CmdCid extends CmdBase implements UsageAware {

private List<SubCmd> commands = new ArrayList<>()

private CidOperation operations
private CidCommand operation

private ConfigMap config

Expand Down Expand Up @@ -81,9 +84,11 @@ class CmdCid extends CmdBase implements UsageAware {
.setOptions(launcher.options)
.setBaseDir(Paths.get('.'))
.build()
// init plugins
Plugins.load(config)
// load the command operations
this.operations = ServiceLoader.load(CidOperation.class).findFirst().orElse(null)
if( !operations )
this.operation = Plugins.getExtension(CidCommand)
if( !operation )
throw new IllegalStateException("Unable to load CID plugin")
// consume the first argument
getCmd(args).apply(args.drop(1))
Expand Down Expand Up @@ -157,7 +162,7 @@ class CmdCid extends CmdBase implements UsageAware {
usage()
return
}
operations.log(config)
operation.log(config)
}

@Override
Expand Down Expand Up @@ -186,7 +191,7 @@ class CmdCid extends CmdBase implements UsageAware {
return
}

operations.show(config, args)
operation.show(config, args)
}

@Override
Expand All @@ -213,7 +218,7 @@ class CmdCid extends CmdBase implements UsageAware {
return
}

operations.lineage(config, args)
operation.lineage(config, args)
}

@Override
Expand Down
10 changes: 9 additions & 1 deletion modules/nextflow/src/test/groovy/nextflow/cli/CmdCidTest.groovy
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2013-2024, Seqera Labs
* Copyright 2013-2025, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -12,6 +12,7 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package nextflow.cli
Expand All @@ -20,6 +21,7 @@ import groovy.json.JsonOutput

import java.nio.file.Files

import nextflow.SysEnv
import nextflow.dag.MermaidHtmlRenderer
import nextflow.data.cid.CidHistoryRecord
import nextflow.data.cid.CidStoreFactory
Expand All @@ -36,9 +38,15 @@ import test.OutputCapture
*/
class CmdCidTest extends Specification {

def setup() {
// clear the environment to avoid the local env pollute the test env
SysEnv.push([:])
}

def cleanup() {
Plugins.stop()
CidStoreFactory.reset()
SysEnv.pop()
}

def setupSpec() {
Expand Down
12 changes: 2 additions & 10 deletions modules/nf-cid/src/main/nextflow/data/cid/CidStore.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,21 @@

package nextflow.data.cid

import java.nio.file.Path

import groovy.transform.CompileStatic
import nextflow.data.config.DataConfig

/**
* Interface for the CID store
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
interface CidStore {
interface CidStore extends Closeable {

/**
* Open the CID store.
* @param config Configuration to open the CID store.
*/
void open(DataConfig config)
CidStore open(DataConfig config)

/**
* Save a CID entry in the store for in a given key.
Expand All @@ -49,12 +47,6 @@ interface CidStore {
*/
Object load(String key)

/**
* Get the CID store location path.
* @return CID store location path.
*/
Path getPath()

/**
* Get the {@link CidHistoryLog} object associated to the CidStore.
* @return {@link CidHistoryLog} object
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,16 @@ abstract class CidStoreFactory implements ExtensionPoint {

private static boolean initialized

protected abstract boolean canOpen(DataConfig config)

protected abstract CidStore newInstance(DataConfig config)

static CidStore create(DataConfig config){
final all = Plugins.getPriorityExtensions(CidStoreFactory)
if( !all )
final factory = Plugins
.getPriorityExtensions(CidStoreFactory)
.find( f-> f.canOpen(config))
if( !factory )
throw new IllegalStateException("Unable to find Nextflow CID store factory")
final factory = all.first()
log.debug "Using Nextflow CID store factory: ${factory.getClass().getName()}"
return factory.newInstance(config)
}
Expand Down
32 changes: 24 additions & 8 deletions modules/nf-cid/src/main/nextflow/data/cid/DefaultCidStore.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package nextflow.data.cid

import nextflow.file.FileHelper
import nextflow.util.TestOnly

import java.nio.file.Files
Expand All @@ -39,17 +40,25 @@ class DefaultCidStore implements CidStore {
private static String HISTORY_FILE_NAME =".history"
private static final String METADATA_FILE = '.data.json'
private static final String METADATA_PATH = '.meta'

private Path metaLocation
private Path location
private CidHistoryLog historyLog

void open(DataConfig config) {
location = config.store.location
metaLocation = config.store.location.resolve(METADATA_PATH)
DefaultCidStore open(DataConfig config) {
location = toLocationPath(config.store.location)
metaLocation = location.resolve(METADATA_PATH)
if( !Files.exists(metaLocation) && !Files.createDirectories(metaLocation) ) {
throw new AbortOperationException("Unable to create CID store directory: $metaLocation")
}
historyLog = new CidHistoryFile(config.store.logLocation ?: metaLocation.resolve(HISTORY_FILE_NAME))
historyLog = new CidHistoryFile(metaLocation.resolve(HISTORY_FILE_NAME))
return this
}

protected Path toLocationPath(String location) {
return location
? FileHelper.toCanonicalPath(location)
: Path.of('.').toAbsolutePath().normalize().resolve('data')
}

@Override
Expand All @@ -70,13 +79,20 @@ class DefaultCidStore implements CidStore {
return null
}

@Override
Path getPath(){ location }
Path getLocation(){
return location
}

@TestOnly
Path getMetadataPath() {metaLocation}
Path getMetadataPath() {
return metaLocation
}

@Override
CidHistoryLog getHistoryLog(){ historyLog }
CidHistoryLog getHistoryLog(){
return historyLog
}

@Override
void close() { }
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
*/
package nextflow.data.cid

import java.util.regex.Pattern

import groovy.transform.CompileStatic
import nextflow.data.config.DataConfig
import nextflow.plugin.Priority
Expand All @@ -29,11 +31,22 @@ import nextflow.plugin.Priority
@Priority(0)
class DefaultCidStoreFactory extends CidStoreFactory {

private static Pattern SCHEME = ~/^([a-zA-Z][a-zA-Z\d+\-.]*):/
private static List SUPPORTED_SCHEMES = ['file', 's3', 'gs', 'az']

@Override
boolean canOpen(DataConfig config) {
final loc = config.store.location
if( !loc ) {
return true
}
final matcher = SCHEME.matcher(loc)
return matcher.find() ? matcher.group(1) in SUPPORTED_SCHEMES : true
}

@Override
protected CidStore newInstance(DataConfig config) {
final cidStore = new DefaultCidStore()
cidStore.open(config)
return cidStore
return new DefaultCidStore() .open(config)
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
*
*/

package nextflow.data.cid.operation
package nextflow.data.cid.cli

import static nextflow.data.cid.fs.CidPath.*

Expand All @@ -34,11 +34,12 @@ import nextflow.data.cid.CidStoreFactory
import nextflow.data.cid.model.DataType
import nextflow.ui.TableBuilder
/**
* Implements CID command line operations
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
@CompileStatic
class CidOperationImpl implements CmdCid.CidOperation {
class CidCommandImpl implements CmdCid.CidCommand {

@Canonical
static class Edge {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@ package nextflow.data.config
import groovy.transform.CompileStatic
import nextflow.Global
import nextflow.Session
import nextflow.util.TestOnly

/**
* Model workflow data config
Expand Down
12 changes: 2 additions & 10 deletions modules/nf-cid/src/main/nextflow/data/config/DataStoreOpts.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,6 @@

package nextflow.data.config

import nextflow.file.FileHelper

import java.nio.file.Path

import groovy.transform.CompileStatic
/**
* Model data store options
Expand All @@ -30,14 +26,10 @@ import groovy.transform.CompileStatic
@CompileStatic
class DataStoreOpts {

final Path location
final Path logLocation
final String location

DataStoreOpts(Map opts) {
this.location = opts.location
? FileHelper.toCanonicalPath(opts.location as String)
: Path.of('.').toAbsolutePath().normalize().resolve('data')
this.logLocation = opts.logLocation ? FileHelper.toCanonicalPath(opts.logLocation as String) : null
this.location = opts.location as String
}

}
2 changes: 1 addition & 1 deletion modules/nf-cid/src/resources/META-INF/extensions.idx
Original file line number Diff line number Diff line change
Expand Up @@ -16,4 +16,4 @@

nextflow.data.cid.DefaultCidStoreFactory
nextflow.data.cid.CidObserverFactory

nextflow.data.cid.cli.CidCommandImpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
/*
* Copyright 2013-2025, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package nextflow.data.cid

import nextflow.data.config.DataConfig
import spock.lang.Specification
import spock.lang.Unroll

/**
*
* @author Paolo Di Tommaso <paolo.ditommaso@gmail.com>
*/
class DefaultCidStoreFactoryTest extends Specification {

@Unroll
def 'should validate can open' () {
given:
def factory = new DefaultCidStoreFactory()
def config = new DataConfig(CONFIG)

expect:
factory.canOpen(config) == EXPECTED

where:
EXPECTED | CONFIG
true | [:]
true | [store:[location:'/some/path']]
true | [store:[location:'some/rel/path']]
true | [store:[location:'file:/this/that']]
true | [store:[location:'s3://some/path']]
false | [store:[location:'http://some/path']]
false | [store:[location:'jdbc:foo']]
}

}
Loading