697.workflow checkout tool

fre/fre.py

@@ -22,7 +22,8 @@
 # click and lazy group loading
 @click.group(
     cls = LazyGroup,
-    lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
+    lazy_subcommands = {"workflow": ".workflow.freworkflow.workflow_cli",
+                       "pp": ".pp.frepp.pp_cli",
                        "catalog": ".catalog.frecatalog.catalog_cli",
                        "list": ".list_.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",

fre/workflow/__init__.py

@@ -0,0 +1,27 @@
+from typing import Optional
+
+def make_workflow_name(experiment : Optional[str] = None) -> str:
+    """
+    Function that takes in a triplet of tags for a model experiment, platform, and target, and
+    returns a directory name for the corresponding pp workflow. Because this is often given by
+    user to the shell being used by python, we split/reform the string to remove semi-colons or
+    spaces that may be used to execute an arbitrary command with elevated privileges.
+
+    :param experiment: One of the postprocessing experiment names from the yaml displayed by fre list exps -y $yamlfile (e.g. c96L65_am5f4b4r0_amip), default None
+    :type experiment: str
+    :param platform: The location + compiler that was used to run the model (e.g. gfdl.ncrc5-deploy), default None
+    :type platform: str
+    :param target: Options used for the model compiler (e.g. prod-openmp), default None
+    :type target: str
+    :return: string created in specific format from the input strings
+    :rtype: str
+
+    .. note:: if any arguments are None, then "None" will appear in the workflow name
+    """
+    name = f'{experiment}__{platform}__{target}'
+    return ''.join(
+                      (''.join(
+                          name.split(' ')
+                              )
+                      ).split(';')
+                  ) # user-input sanitation, prevents some malicious cmds from being executed with privileges

fre/workflow/checkout_script.py

@@ -0,0 +1,171 @@
+""" Workflow checkout """
+import os
+import subprocess
+from pathlib import Path
+import logging
+import shutil 
+
+import fre.yamltools.combine_yamls_script as cy
+from fre.app.helpers import change_directory
+#from . import make_workflow_name
+from jsonschema import validate, SchemaError, ValidationError
+
+fre_logger = logging.getLogger(__name__)
+
+FRE_PP_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git"
+FRE_RUN_WORKFLOW = "https://github.com/NOAA-GFDL/fre-workflows.git"
+
+######VALIDATE#####
+def validate_yaml(yamlfile: dict, application: str) -> None:
+    """
+    Validate the format of the yaml file based
+    on the schema.json in gfdl_msd_schemas
+
+    :param yamlfile: Model, settings, pp, and analysis yaml
+                     information combined into a dictionary
+    :type yamlfile: dict
+    :param application: ------------------------------------------------
+    :type application: string
+    :raises ValueError:
+        - if gfdl_mdf_schema path is not valid
+        - combined yaml is not valid
+        - unclear error in validation
+    :return: None
+    :rtype: None
+    """
+    schema_dir = Path(__file__).resolve().parents[1]
+    schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', f'fre_{application}.json')
+    fre_logger.info("Using yaml schema '%s'", schema_path)
+    # Load the json schema: .load() (vs .loads()) reads and parses the json in one)
+    try:
+        with open(schema_path,'r', encoding='utf-8') as s:
+            schema = json.load(s)
+    except:
+        fre_logger.error("Schema '%s' is not valid. Contact the FRE team.", schema_path)
+        raise
+
+    # Validate yaml
+    # If the yaml is not valid, the schema validation will raise errors and exit
+    try:
+        validate(instance = yamlfile,schema=schema)
+        fre_logger.info("Combined yaml valid")
+    except SchemaError as exc:
+        raise ValueError(f"Schema '{schema_path}' is not valid. Contact the FRE team.") from exc
+    except ValidationError as exc:
+        raise ValueError("Combined yaml is not valid. Please fix the errors and try again.") from exc
+    except Exception as exc:
+        raise ValueError("Unclear error from validation. Please try to find the error and try again.") from exc
+
+def workflow_checkout(yamlfile: str = None, experiment = None,
+                      application = None, branch = None):
+    """
+    Create a directory and clone the workflow template files from a defined repo.
+
+    :param yamlfile: Model yaml configuration file
+    :type yamlfile: str
+    :param experiment: One of the postprocessing experiment names from the
+                       yaml displayed by fre list exps -y $yamlfile 
+                       (e.g. c96L65_am5f4b4r0_amip), default None
+    :type experiment: str
+    :param platform: The location + compiler that was used to run the model
+                     (e.g. gfdl.ncrc5-deploy), default None
+    :type platform: str
+    :param target: Options used for the model compiler (e.g. prod-openmp), default None
+    :type target: str
+    :param branch: which git branch to pull from, default None
+    :type branch: str
+    :param application: Which workflow will be used/cloned
+    :type application: str
+    :raises OSError: why checkout script was not able to be created
+    :raises ValueError:
+        -if experiment or platform or target is None
+        -if branch argument cannot be found as a branch or tag
+    """
+    # Used in consolidate_yamls function for now
+    platform = None
+    target = None
+    if application == "run":
+        fre_logger.info("NOT DONE YET")
+        # will probably be taken out and put above is "use"
+        # is generalized in this tool
+        yaml = cy.consolidate_yamls(yamlfile=yamlfile,
+                                    experiment=experiment,
+                                    platform=platform,
+                                    target=target,
+                                    use="run",
+                                    output=None)
+        #validate_yaml(yamlfile = yaml, application = "run")
+        workflow_info = yaml.get("workflow").get("run_workflow")
+    elif application == "pp":
+        # will probably be taken out and put above is "use"
+        # is generalized in this tool
+        yaml = cy.consolidate_yamls(yamlfile=yamlfile,
+                                    experiment=experiment,
+                                    platform=platform,
+                                    target=target,
+                                    use="pp",
+                                    output=f"config.yaml")
+        #validate_yaml(yamlfile = yaml, application = "pp")
+        workflow_info = yaml.get("workflow").get("pp_workflow")
+
+    repo = workflow_info.get("repo")
+
+    if not branch:
+        tag = workflow_info.get("version")
+        fre_logger.info("Default tag ==> '%s'", tag)
+    else:
+        tag = branch
+        fre_logger.info("Requested branch/tag ==> '%s'", tag)
+
+    if None in [repo, tag]:
+        raise ValueError(f"One of these are None: repo / tag = {repo} / {tag}")
+
+    fre_logger.warning("(%s):(%s) check out ==> REQUESTED", repo, tag)
+
+    # clone directory
+    directory = os.path.expanduser("~/cylc-src")
+    # workflow name
+    workflow_name = experiment
+
+    # create workflow in cylc-src
+    try:
+        Path(directory).mkdir(parents=True, exist_ok=True)
+    except Exception as exc:
+        raise OSError(
+            f"(checkoutScript) directory {directory} wasn't able to be created. exit!") from exc
+
+    if not Path(f"{directory}/{workflow_name}").is_dir():
+        # scenarios 1+2, checkout doesn't exist, branch specified (or not)
+        fre_logger.info("Workflow does not yet exist; will create now")
+        clone_output = subprocess.run( ["git", "clone","--recursive",
+                                        f"--branch={tag}",
+                                        repo, f"{directory}/{workflow_name}"],
+                                        capture_output = True, text = True, check = True)
+        fre_logger.debug(clone_output)
+        fre_logger.warning("(%s):(%s) check out ==> SUCCESSFUL", repo, tag)
+    else:
+        # the repo checkout does exist, scenarios 3 and 4.
+        with change_directory(f"{directory}/{workflow_name}"):
+            # capture the branch and tag
+            # if either match git_clone_branch_arg, then success. otherwise, fail.
+            current_tag = subprocess.run(["git","describe","--tags"],
+                                         capture_output = True,
+                                         text = True, check = True).stdout.strip()
+            current_branch = subprocess.run(["git", "branch", "--show-current"],
+                                             capture_output = True,
+                                             text = True, check = True).stdout.strip()
+
+            if tag in (current_tag, current_branch):
+                fre_logger.warning("Checkout exists ('%s/%s'), and matches '%s'", directory, workflow_name, tag)
+            else:
+                fre_logger.error(
+                    "ERROR: Checkout exists ('%s/%s') and does not match '%s'", directory, workflow_name, tag)
+                fre_logger.error(
+                    "ERROR: Current branch: '%s', Current tag-describe: '%s'", current_branch, current_tag)
+                raise ValueError('Neither tag nor branch matches the git clone branch arg')
+
+    ## Move combined yaml to cylc-src location
+    cylc_src_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{experiment}")
+#    outfile = os.path.join(cylc_src_dir, f"{experiment}.yaml")
+    shutil.move(f"config.yaml", cylc_src_dir)
+    fre_logger.info("Combined yaml file moved to ~/cylc-src/%s", experiment)

fre/workflow/freworkflow.py

@@ -0,0 +1,79 @@
+''' fre workflow '''
+
+import click
+import logging
+fre_logger = logging.getLogger(__name__)
+
+#fre tools
+from . import checkout_script
+#from . import install_script
+#from . import run_script
+
+@click.group(help=click.style(" - workflow subcommands", fg=(57,139,210)))
+def workflow_cli():
+    ''' entry point to fre workflow click commands '''
+
+@workflow_cli.command()
+@click.option("-y", "--yamlfile", type=str,
+              help="Model yaml file",
+              required=True)
+@click.option("-e", "--experiment", type=str,
+              help="Experiment name",
+              required=True)
+@click.option("-b", "--branch", type =str,
+              required=False, default = None,
+              help="fre-workflows branch/tag to clone; default is $(fre --version)")
+@click.option("-a", "--application",
+              type=click.Choice(['run', 'pp']),
+              help="Use case for checked out workflow",
+              required=True)
+def checkout(yamlfile, experiment, application, branch=None):
+    """
+    Checkout/extract fre workflow
+    """
+    checkout_script.workflow_checkout(yamlfile, experiment, application, branch)
+
+#@workflow_cli.command()
+#@click.option("-e", "--experiment", type=str,
+#              help="Experiment name",
+#              required=True)
+#def install(experiment):
+#    """
+#    Install workflow configuration
+#    """
+#    install_script.workflow_install(experiment)
+
+#@workflow_cli.command()
+#@click.option("-e", "--experiment", type=str,
+#              help="Experiment name",
+#              required=True)
+#@click.option("--pause", is_flag=True, default=False,
+#              help="Pause the workflow immediately on start up",
+#              required=False)
+#@click.option("--no_wait", is_flag=True, default=False,
+#              help="after submission, do not wait to ping the scheduler and confirm success",
+#              required=False)
+#def run(experiment, pause, no_wait):
+#    """
+#    Run workflow configuration
+#    """
+#    run_script.workflow_run(experiment, pause, no_wait)
+
+#@workflow_cli.command()
+#@click.option("-e", "--experiment", type=str,
+#              help="Experiment name",
+#              required=True)
+#@click.option("-c", "--config-file", type=str,
+#              help="Path to a configuration file in either XML or YAML",
+#              required=True)
+#@click.option("-b", "--branch",
+#              required=False, default=None,
+#              help="fre-workflows branch/tag to clone; default is $(fre --version)")
+#@click.option("-t", "--time",
+#              required=False, default=None,
+#              help="Time whose history files are ready")
+#def all(experiment, platform, target, config_file, branch, time):
+#    """
+#    Execute all fre workflow initialization steps in order
+#    """
+#    wrapper_script.run_workflow_steps(experiment, platform, target, config_file, branch, time)

fre/workflow/tests/AM5_example/am5.yaml

@@ -0,0 +1,71 @@
+# reusable variables
+fre_properties:
+  - &AM5_VERSION        "am5f7b12r1"
+  - &FRE_STEM           !join [am5/, *AM5_VERSION]
+
+  # amip
+  - &EXP_AMIP_START     "19790101T0000Z"
+  - &EXP_AMIP_END       "20200101T0000Z"
+  - &ANA_AMIP_START     "19800101T0000Z"
+  - &ANA_AMIP_END       "20200101T0000Z"
+
+  - &PP_AMIP_CHUNK96    "P1Y"
+  - &PP_AMIP_CHUNK384   "P1Y"
+  - &PP_XYINTERP96      "180,288"
+  - &PP_XYINTERP384     "720,1152"
+
+  # climo
+  - &EXP_CLIMO_START96  "0001"
+  - &EXP_CLIMO_END96    "0011"
+  - &ANA_CLIMO_START96  "0002"
+  - &ANA_CLIMO_END96    "0011"
+
+  - &EXP_CLIMO_START384 "0001"
+  - &EXP_CLIMO_END384   "0006"
+  - &ANA_CLIMO_START384 "0002"
+  - &ANA_CLIMO_END384   "0006"
+
+  # coupled
+  - &PP_CPLD_CHUNK_A    "P5Y"
+  - &PP_CPLD_CHUNK_B    "P20Y"
+
+  # grids
+  - &GRID_SPEC96         "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
+
+  # compile information
+  - &release             "f1a1r1"
+  - &INTEL               "intel-classic"
+  - &FMSincludes         "-IFMS/fms2_io/include -IFMS/include -IFMS/mpp/include"
+  - &momIncludes         "-Imom6/MOM6-examples/src/MOM6/pkg/CVMix-src/include"
+
+# compile information
+build:
+  compileYaml: "compile.yaml"
+  platformYaml: "yaml_include/platforms.yaml"
+
+experiments:
+  - name: "c96L65_am5f7b12r1_amip_TESTING"
+    settings: "yaml_include/settings.yaml"
+    pp:
+      - "yaml_include/pp.c96_amip.yaml"
+      - "yaml_include/pp-test.c96_amip.yaml"
+  - name: "c96L65_am5f7b12r1_amip_TESTING_WRONG"
+    settings: "yaml_include/settings_WRONG.yaml"
+    pp:
+      - "yaml_include/pp.c96_amip.yaml"
+  - name: "c96L65_am5f7b12r1_pdclim1850F"
+    pp:
+      - "yaml_include/pp.c96_clim.yaml"
+
+#  amip:
+#    settings:
+#      - shared/settings.yaml
+#      - shared/directories.yaml
+#    run:
+#      version: 1.1
+#      - run/inputs.yaml
+#      - run/runtime.yaml
+#    postprocess:
+#      version: 2.0
+#      - pp/components
+#      - analysis/legacy-bw.yaml

fre/workflow/tests/AM5_example/yaml_include/pp-test.c96_amip.yaml

@@ -0,0 +1,33 @@
+# local reusable variable overrides
+fre_properties:
+  - &custom_interp "200,200"
+
+#c96_amip_postprocess:
+postprocess:
+  components:
+    - type: "atmos_cmip-TEST"
+      sources:
+        - history_file: "atmos_month_cmip"
+        - history_file: "atmos_8xdaily_cmip"
+        - history_file: "atmos_daily_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *custom_interp
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos-TEST"
+      sources:
+        - history_file: "atmos_month"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False
+    - type: "atmos_level_cmip-TEST"
+      sources:
+        - history_file: "atmos_level_cmip"
+      sourceGrid: "cubedsphere"
+      xyInterp: *PP_XYINTERP96
+      interpMethod: "conserve_order2"
+      inputRealm: 'atmos'
+      postprocess_on: False