From bb4e34586caed36f3da5f5f9420a8deeb92f9467 Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 10 Jan 2024 10:55:38 +0100 Subject: [PATCH 1/3] runcrate add: support for setting properties --- rocrate/cli.py | 42 ++++++++++++++++++++++++++++++------------ rocrate/rocrate.py | 14 ++++++++------ test/test_cli.py | 29 +++++++++++++++++++++-------- 3 files changed, 59 insertions(+), 26 deletions(-) diff --git a/rocrate/cli.py b/rocrate/cli.py index 8ff4e88..377873a 100644 --- a/rocrate/cli.py +++ b/rocrate/cli.py @@ -44,8 +44,20 @@ def convert(self, value, param, ctx): self.fail(f"{value!r} is not splittable", param, ctx) +class KeyValueParamType(click.ParamType): + name = "key_value" + + def convert(self, value, param, ctx): + try: + return tuple(value.split("=", 1)) if value else () + except AttributeError: + self.fail(f"{value!r} is not splittable", param, ctx) + + CSV = CSVParamType() +KeyValue = KeyValueParamType() OPTION_CRATE_PATH = click.option('-c', '--crate-dir', type=click.Path(), default=os.getcwd) +OPTION_PROPS = click.option('-P', '--property', type=KeyValue, multiple=True) @click.group() @@ -72,7 +84,8 @@ def add(): @add.command() @click.argument('path', type=click.Path(exists=True, dir_okay=False)) @OPTION_CRATE_PATH -def file(crate_dir, path): +@OPTION_PROPS +def file(crate_dir, path, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) source = Path(path).resolve(strict=True) try: @@ -80,14 +93,15 @@ def file(crate_dir, path): except ValueError: # For now, only support adding an existing file to the metadata raise ValueError(f"{source} is not in the crate dir {crate_dir}") - crate.add_file(source, dest_path) + crate.add_file(source, dest_path, properties=dict(property)) crate.metadata.write(crate_dir) @add.command() @click.argument('path', type=click.Path(exists=True, file_okay=False)) @OPTION_CRATE_PATH -def dataset(crate_dir, path): +@OPTION_PROPS +def dataset(crate_dir, path, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) source = Path(path).resolve(strict=True) try: @@ -95,7 +109,7 @@ def dataset(crate_dir, path): except ValueError: # For now, only support adding an existing directory to the metadata raise ValueError(f"{source} is not in the crate dir {crate_dir}") - crate.add_dataset(source, dest_path) + crate.add_dataset(source, dest_path, properties=dict(property)) crate.metadata.write(crate_dir) @@ -103,7 +117,8 @@ def dataset(crate_dir, path): @click.argument('path', type=click.Path(exists=True)) @click.option('-l', '--language', type=click.Choice(LANG_CHOICES), default="cwl") @OPTION_CRATE_PATH -def workflow(crate_dir, path, language): +@OPTION_PROPS +def workflow(crate_dir, path, language, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) source = Path(path).resolve(strict=True) try: @@ -112,7 +127,7 @@ def workflow(crate_dir, path, language): # For now, only support marking an existing file as a workflow raise ValueError(f"{source} is not in the crate dir {crate_dir}") # TODO: add command options for main and gen_cwl - crate.add_workflow(source, dest_path, main=True, lang=language, gen_cwl=False) + crate.add_workflow(source, dest_path, main=True, lang=language, gen_cwl=False, properties=dict(property)) crate.metadata.write(crate_dir) @@ -121,9 +136,10 @@ def workflow(crate_dir, path, language): @click.option('-n', '--name') @click.option('-m', '--main-entity') @OPTION_CRATE_PATH -def suite(crate_dir, identifier, name, main_entity): +@OPTION_PROPS +def suite(crate_dir, identifier, name, main_entity, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) - suite = crate.add_test_suite(identifier=add_hash(identifier), name=name, main_entity=main_entity) + suite = crate.add_test_suite(identifier=add_hash(identifier), name=name, main_entity=main_entity, properties=dict(property)) crate.metadata.write(crate_dir) print(suite.id) @@ -136,11 +152,12 @@ def suite(crate_dir, identifier, name, main_entity): @click.option('-i', '--identifier') @click.option('-n', '--name') @OPTION_CRATE_PATH -def instance(crate_dir, suite, url, resource, service, identifier, name): +@OPTION_PROPS +def instance(crate_dir, suite, url, resource, service, identifier, name, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) instance_ = crate.add_test_instance( add_hash(suite), url, resource=resource, service=service, - identifier=add_hash(identifier), name=name + identifier=add_hash(identifier), name=name, properties=dict(property) ) crate.metadata.write(crate_dir) print(instance_.id) @@ -152,7 +169,8 @@ def instance(crate_dir, suite, url, resource, service, identifier, name): @click.option('-e', '--engine', type=click.Choice(ENGINE_CHOICES), default="planemo") @click.option('-v', '--engine-version') @OPTION_CRATE_PATH -def definition(crate_dir, suite, path, engine, engine_version): +@OPTION_PROPS +def definition(crate_dir, suite, path, engine, engine_version, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) source = Path(path).resolve(strict=True) try: @@ -162,7 +180,7 @@ def definition(crate_dir, suite, path, engine, engine_version): raise ValueError(f"{source} is not in the crate dir {crate_dir}") crate.add_test_definition( add_hash(suite), source=source, dest_path=dest_path, engine=engine, - engine_version=engine_version + engine_version=engine_version, properties=dict(property) ) crate.metadata.write(crate_dir) diff --git a/rocrate/rocrate.py b/rocrate/rocrate.py index 0bf2166..daed143 100644 --- a/rocrate/rocrate.py +++ b/rocrate/rocrate.py @@ -497,23 +497,24 @@ def add_workflow( workflow.subjectOf = cwl_workflow return workflow - def add_test_suite(self, identifier=None, name=None, main_entity=None): + def add_test_suite(self, identifier=None, name=None, main_entity=None, properties=None): test_ref_prop = "mentions" if not main_entity: main_entity = self.mainEntity if not main_entity: test_ref_prop = "about" - suite = self.add(TestSuite(self, identifier)) - suite.name = name or suite.id.lstrip("#") + suite = self.add(TestSuite(self, identifier, properties=properties)) + if not properties or "name" not in properties: + suite.name = name or suite.id.lstrip("#") if main_entity: suite["mainEntity"] = main_entity self.root_dataset.append_to(test_ref_prop, suite) self.metadata.extra_terms.update(TESTING_EXTRA_TERMS) return suite - def add_test_instance(self, suite, url, resource="", service="jenkins", identifier=None, name=None): + def add_test_instance(self, suite, url, resource="", service="jenkins", identifier=None, name=None, properties=None): suite = self.__validate_suite(suite) - instance = self.add(TestInstance(self, identifier)) + instance = self.add(TestInstance(self, identifier, properties=properties)) instance.url = url instance.resource = resource if isinstance(service, TestService): @@ -522,7 +523,8 @@ def add_test_instance(self, suite, url, resource="", service="jenkins", identifi service = get_service(self, service) self.add(service) instance.service = service - instance.name = name or instance.id.lstrip("#") + if not properties or "name" not in properties: + instance.name = name or instance.id.lstrip("#") suite.append_to("instance", instance) self.metadata.extra_terms.update(TESTING_EXTRA_TERMS) return instance diff --git a/test/test_cli.py b/test/test_cli.py index b6b8b69..28d918e 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -145,18 +145,19 @@ def test_cli_add_file(tmpdir, test_data_dir, helpers, monkeypatch, cwd): # add shutil.copy(test_data_dir / "sample_file.txt", crate_dir) file_path = crate_dir / "sample_file.txt" - args = ["add", "file"] + args = ["add", "file", str(file_path), "-P", "name=foo", "-P", "description=foo bar"] if cwd: monkeypatch.chdir(str(crate_dir)) file_path = file_path.relative_to(crate_dir) else: args.extend(["-c", str(crate_dir)]) - args.append(str(file_path)) result = runner.invoke(cli, args) assert result.exit_code == 0 json_entities = helpers.read_json_entities(crate_dir) assert "sample_file.txt" in json_entities assert json_entities["sample_file.txt"]["@type"] == "File" + assert json_entities["sample_file.txt"]["name"] == "foo" + assert json_entities["sample_file.txt"]["description"] == "foo bar" @pytest.mark.parametrize("cwd", [False, True]) @@ -171,18 +172,19 @@ def test_cli_add_dataset(tmpdir, test_data_dir, helpers, monkeypatch, cwd): # add dataset_path = crate_dir / "test_add_dir" shutil.copytree(test_data_dir / "test_add_dir", dataset_path) - args = ["add", "dataset"] + args = ["add", "dataset", str(dataset_path), "-P", "name=foo", "-P", "description=foo bar"] if cwd: monkeypatch.chdir(str(crate_dir)) dataset_path = dataset_path.relative_to(crate_dir) else: args.extend(["-c", str(crate_dir)]) - args.append(str(dataset_path)) result = runner.invoke(cli, args) assert result.exit_code == 0 json_entities = helpers.read_json_entities(crate_dir) assert "test_add_dir/" in json_entities assert json_entities["test_add_dir/"]["@type"] == "Dataset" + assert json_entities["test_add_dir/"]["name"] == "foo" + assert json_entities["test_add_dir/"]["description"] == "foo bar" @pytest.mark.parametrize("cwd", [False, True]) @@ -196,7 +198,7 @@ def test_cli_add_workflow(test_data_dir, helpers, monkeypatch, cwd): assert json_entities["sort-and-change-case.ga"]["@type"] == "File" # add wf_path = crate_dir / "sort-and-change-case.ga" - args = ["add", "workflow"] + args = ["add", "workflow", "-P", "name=foo", "-P", "description=foo bar"] if cwd: monkeypatch.chdir(str(crate_dir)) wf_path = wf_path.relative_to(crate_dir) @@ -212,6 +214,8 @@ def test_cli_add_workflow(test_data_dir, helpers, monkeypatch, cwd): lang_id = f"https://w3id.org/workflowhub/workflow-ro-crate#{lang}" assert lang_id in json_entities assert json_entities["sort-and-change-case.ga"]["programmingLanguage"]["@id"] == lang_id + assert json_entities["sort-and-change-case.ga"]["name"] == "foo" + assert json_entities["sort-and-change-case.ga"]["description"] == "foo bar" @pytest.mark.parametrize("cwd", [False, True]) @@ -228,20 +232,27 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd): wf_path = crate_dir / "sort-and-change-case.ga" assert runner.invoke(cli, ["add", "workflow", "-c", str(crate_dir), "-l", "galaxy", str(wf_path)]).exit_code == 0 # add test suite - result = runner.invoke(cli, ["add", "test-suite", "-c", str(crate_dir)]) + result = runner.invoke(cli, ["add", "test-suite", "-c", str(crate_dir), + "-P", "name=foo", "-P", "description=foo bar"]) assert result.exit_code == 0 suite_id = result.output.strip() json_entities = helpers.read_json_entities(crate_dir) assert suite_id in json_entities + assert json_entities[suite_id]["name"] == "foo" + assert json_entities[suite_id]["description"] == "foo bar" # add test instance - result = runner.invoke(cli, ["add", "test-instance", "-c", str(crate_dir), suite_id, "http://example.com", "-r", "jobs"]) + result = runner.invoke(cli, ["add", "test-instance", "-c", str(crate_dir), + suite_id, "http://example.com", "-r", "jobs", + "-P", "name=foo", "-P", "description=foo bar"]) assert result.exit_code == 0 instance_id = result.output.strip() json_entities = helpers.read_json_entities(crate_dir) assert instance_id in json_entities + assert json_entities[instance_id]["name"] == "foo" + assert json_entities[instance_id]["description"] == "foo bar" # add test definition def_path = crate_dir / def_id - args = ["add", "test-definition"] + args = ["add", "test-definition", "-P", "name=foo", "-P", "description=foo bar"] if cwd: monkeypatch.chdir(str(crate_dir)) def_path = def_path.relative_to(crate_dir) @@ -253,6 +264,8 @@ def test_cli_add_test_metadata(test_data_dir, helpers, monkeypatch, cwd): json_entities = helpers.read_json_entities(crate_dir) assert def_id in json_entities assert set(json_entities[def_id]["@type"]) == {"File", "TestDefinition"} + assert json_entities[def_id]["name"] == "foo" + assert json_entities[def_id]["description"] == "foo bar" # check extra terms metadata_path = crate_dir / helpers.METADATA_FILE_NAME with open(metadata_path, "rt") as f: From 1b5d67f96f068ff9d28618c32558c5be27ba56ec Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 10 Jan 2024 13:26:35 +0100 Subject: [PATCH 2/3] update CLI docs --- README.md | 27 ++++++++++++++++++++++++++- rocrate/cli.py | 2 +- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index dbbe578..76aa7e2 100644 --- a/README.md +++ b/README.md @@ -310,7 +310,7 @@ The command acts on the current directory, unless the `-c` option is specified. ### Adding items to the crate -The `rocrate add` command allows to add workflows and other entity types (currently [testing-related metadata](https://crs4.github.io/life_monitor/workflow_testing_ro_crate)) to an RO-Crate: +The `rocrate add` command allows to add file, datasets (directories), workflows and other entity types (currently [testing-related metadata](https://crs4.github.io/life_monitor/workflow_testing_ro_crate)) to an RO-Crate: ```console $ rocrate add --help @@ -320,6 +320,8 @@ Options: --help Show this message and exit. Commands: + dataset + file test-definition test-instance test-suite @@ -372,6 +374,29 @@ rocrate add test-instance test1 http://example.com -r jobs -i test1_1 rocrate add test-definition test1 test/test1/sort-and-change-case-test.yml -e planemo -v '>=0.70' ``` +To add files or directories after crate initialization: + +```bash +cp ../sample_file.txt . +rocrate add file sample_file.txt -P name=sample -P description="Sample file" +cp -r ../test_add_dir . +rocrate add dataset test_add_dir +``` + +The above example also shows how to set arbitrary properties for the entity with `-P`. This is supported by most `rocrate add` subcommands. + +```console +$ rocrate add workflow --help +Usage: rocrate add workflow [OPTIONS] PATH + +Options: + -l, --language [cwl|galaxy|knime|nextflow|snakemake|compss|autosubmit] + -c, --crate-dir PATH + -P, --property KEY=VALUE + --help Show this message and exit. +``` + + ## License * Copyright 2019-2024 The University of Manchester, UK diff --git a/rocrate/cli.py b/rocrate/cli.py index 377873a..1f5ee06 100644 --- a/rocrate/cli.py +++ b/rocrate/cli.py @@ -57,7 +57,7 @@ def convert(self, value, param, ctx): CSV = CSVParamType() KeyValue = KeyValueParamType() OPTION_CRATE_PATH = click.option('-c', '--crate-dir', type=click.Path(), default=os.getcwd) -OPTION_PROPS = click.option('-P', '--property', type=KeyValue, multiple=True) +OPTION_PROPS = click.option('-P', '--property', type=KeyValue, multiple=True, metavar="KEY=VALUE") @click.group() From 5e2802edf93bee560e3add622cd5a9cbe3bda68f Mon Sep 17 00:00:00 2001 From: simleo Date: Wed, 10 Jan 2024 13:37:57 +0100 Subject: [PATCH 3/3] fix line too long --- rocrate/cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rocrate/cli.py b/rocrate/cli.py index 1f5ee06..b4ff19e 100644 --- a/rocrate/cli.py +++ b/rocrate/cli.py @@ -139,7 +139,10 @@ def workflow(crate_dir, path, language, property): @OPTION_PROPS def suite(crate_dir, identifier, name, main_entity, property): crate = ROCrate(crate_dir, init=False, gen_preview=False) - suite = crate.add_test_suite(identifier=add_hash(identifier), name=name, main_entity=main_entity, properties=dict(property)) + suite = crate.add_test_suite( + identifier=add_hash(identifier), name=name, main_entity=main_entity, + properties=dict(property) + ) crate.metadata.write(crate_dir) print(suite.id)