feat: refactor dataset create (#379)

## Description

This is a rewrite of the code that parses the recipe and invoke the
sources and filters in the right order. These changes are merely a
simplification of the old version and are 100% backward compatible.
The input family of actions has been refactored such that all the
standalone action files, e.g. concat are consolidated into a single
action.py.

Additional related additions.
- Add facility to have a source as a parameter of an other source if
needed.
- Changes are due to code moving to their own file (i.e repeated_dates).
- A new CLI tool anemoi-datasets recipe was added to check the change.
It can be use to format the YAML and validate it (against a simple JSON
schema), and migrate older YAML files to new syntax. The later is
experimental, and as been tested with very old recipe files. It can be a
good starting point for future migrations, if ever needed.


## What problem does this change solve?

This change is in preparation of adding support for observation in
anemoi-dataset[create].

## What issue or task does this change relate to?
<!-- link to Issue Number -->

##  Additional notes ##
<!-- Include any additional information, caveats, or considerations that
the reviewer should be aware of. -->

***As a contributor to the Anemoi framework, please ensure that your
changes include unit tests, updates to any affected dependencies and
documentation, and have been tested in a parallel setting (i.e., with
multiple GPUs). As a reviewer, you are also responsible for verifying
these aspects and requesting changes if they are not adequately
addressed. For guidelines about those please refer to
https://anemoi.readthedocs.io/en/latest/***

By opening this pull request, I affirm that all authors agree to the
[Contributor License
Agreement.](https://github.com/ecmwf/codex/blob/main/Legal/contributor_license_agreement.md)

---------

Co-authored-by: Florian Pinault <floriankrb@users.noreply.github.com>
Co-authored-by: Matthew Chantry <matthew.chantry@ecmwf.int>
Co-authored-by: Marek Jacob <MeraX@users.noreply.github.com>
Co-authored-by: Aaron Hopkinson <197336788+aaron-hopkinson@users.noreply.github.com>

Author: 5 people

.gitignore

@@ -141,3 +141,13 @@ _version.py
 *.to_upload
 tempCodeRunnerFile.python
 Untitled-*.py
+*.zip
+*.json
+*.db
+*.tgz
+_api/
+trace.txt
+?/
+*.prof
+prof/
+*.gz

pyproject.toml

@@ -51,9 +51,12 @@ dependencies = [
   "anemoi-transform>=0.1.10",
   "anemoi-utils[provenance]>=0.4.32",
   "cfunits",
+  "glom",
+  "jsonschema",
   "numcodecs<0.16",                   # Until we move to zarr3
   "numpy",
   "pyyaml",
+  "ruamel-yaml",
   "semantic-version",
   "tqdm",
   "zarr<=2.18.4",

src/anemoi/datasets/commands/recipe/__init__.py

@@ -0,0 +1,93 @@
+# (C) Copyright 2024 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import argparse
+import logging
+import sys
+from typing import Any
+
+import yaml
+
+from anemoi.datasets.create import validate_config
+
+from .. import Command
+from .format import format_recipe
+from .migrate import migrate_recipe
+
+LOG = logging.getLogger(__name__)
+
+
+class Recipe(Command):
+    def add_arguments(self, command_parser: Any) -> None:
+        """Add arguments to the command parser.
+
+        Parameters
+        ----------
+        command_parser : Any
+            Command parser object.
+        """
+
+        command_parser.add_argument("--validate", action="store_true", help="Validate recipe.")
+        command_parser.add_argument("--format", action="store_true", help="Format the recipe.")
+        command_parser.add_argument("--migrate", action="store_true", help="Migrate the recipe to the latest version.")
+
+        group = command_parser.add_mutually_exclusive_group()
+        group.add_argument("--inplace", action="store_true", help="Overwrite the recipe file in place.")
+        group.add_argument("--output", type=str, help="Output file path for the converted recipe.")
+
+        command_parser.add_argument(
+            "path",
+            help="Path to recipe.",
+        )
+
+    def run(self, args: Any) -> None:
+
+        if not args.validate and not args.format and not args.migrate:
+            args.validate = True
+
+        with open(args.path) as file:
+            config = yaml.safe_load(file)
+
+        assert isinstance(config, dict)
+
+        if args.validate:
+            if args.inplace and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--inplace is not supported with --validate.")
+
+            if args.output and (not args.format and not args.migrate):
+                argparse.ArgumentError(None, "--output is not supported with --validate.")
+
+            validate_config(config)
+            LOG.info(f"{args.path}: Recipe is valid.")
+            return
+
+        if args.migrate:
+            config = migrate_recipe(args, config)
+            if config is None:
+                LOG.info(f"{args.path}: No changes needed.")
+                return
+
+            args.format = True
+
+        if args.format:
+            formatted = format_recipe(args, config)
+            assert "dates" in formatted
+            f = sys.stdout
+            if args.output:
+                f = open(args.output, "w")
+
+            if args.inplace:
+                f = open(args.path, "w")
+
+            print(formatted, file=f)
+            f.close()
+
+
+command = Recipe

src/anemoi/datasets/commands/recipe/format.py

@@ -0,0 +1,55 @@
+# (C) Copyright 2025 Anemoi contributors.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+import datetime
+import logging
+
+from ...dumper import yaml_dump
+
+LOG = logging.getLogger(__name__)
+
+
+def make_dates(config):
+    if isinstance(config, dict):
+        return {k: make_dates(v) for k, v in config.items()}
+    if isinstance(config, list):
+        return [make_dates(v) for v in config]
+    if isinstance(config, str):
+        try:
+            return datetime.datetime.fromisoformat(config)
+        except ValueError:
+            return config
+    return config
+
+
+ORDER = (
+    "name",
+    "description",
+    "dataset_status",
+    "licence",
+    "attribution",
+    "env",
+    "dates",
+    "common",
+    "data_sources",
+    "input",
+    "output",
+    "statistics",
+    "build",
+    "platform",
+)
+
+
+def format_recipe(args, config: dict) -> str:
+
+    config = make_dates(config)
+    assert config
+
+    return yaml_dump(config, order=ORDER)