Enable setup scripts to be callable from elsewhere (#1742)

ci/scripts/create_experiment.py

@@ -2,108 +2,89 @@
 
 """
 Basic python script to create an experiment directory on the fly from a given
-
 yaml file for the arguments to the two scripts below in ${HOMEgfs}/workflow
-
 where ${HOMEgfs} is specified within the input yaml file.
 
  ${HOMEgfs}/workflow/setup_expt.py
  ${HOMEgfs}/workflow/setup_xml.py
 
 The yaml file are simply the arguments for these two scripts.
-After this scripts runs these two the use will have an experiment ready for launching
+After this scripts runs the experiment is ready for launch.
 
 Output
 ------
-
 Functionally an experiment is setup as a result running the two scripts described above
 with an error code of 0 upon success.
 """
 
 import os
+import sys
 from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
 from pathlib import Path
 
-from wxflow import YAMLFile, Logger, logit, Executable
+from wxflow import YAMLFile, Logger, logit
 
 
+_here = os.path.dirname(__file__)
+_top = os.path.abspath(os.path.join(os.path.abspath(_here), '../..'))
 logger = Logger(level='DEBUG', colored_log=True)
 
 
+# TODO: move create_experiment.py to workflow/ and remove this sys.path.insert business
+sys.path.insert(0, os.path.join(_top, 'workflow'))
+import setup_expt
+import setup_xml
+
+
 @logit(logger)
 def input_args():
     """
-    Method to collect user arguments for `create_experiment.py`
-
-    Input
-    -----
-
-    A single key valued argument: --yaml <full path to YAML file>
-
     Description
     -----------
 
-    A full path to a YAML file with the following format with required sections: experiment, arguments
+    Method to collect user arguments for `create_experiment.py`
 
-    experiment:
-        mode: <cycled> <forecast-only>
-            used to hold the only required positional argument to setup_expt.py
+    Parameters
+    ----------
 
-    arguments:
-        holds all the remaining key values pairs for all requisite arguments documented for setup_expt.py
-        Note: the argument pslot is derived from the basename of the yamlfile itself
+    None
 
     Returns
     -------
 
-    args: Namespace
-
-        Namespace with the value of the file path to a yaml file from the key yaml
+    argparse.Namespace:
+        argparse.Namespace with the value of the file path to a yaml file from the key yaml
     """
 
-    description = """Single argument as a yaml file containing the
-    key value pairs as arguments to setup_expt.py
-    """
+    description = """Create a global-workflow experiment"""
 
     parser = ArgumentParser(description=description,
                             formatter_class=ArgumentDefaultsHelpFormatter)
 
-    parser.add_argument('--yaml', help='yaml configuration file per experiment', type=str, required=True)
-    parser.add_argument('--dir', help='full path to top level of repo of global-workflow', type=str, required=True)
+    parser.add_argument('--yaml', help='full path to yaml file describing the experiment configuration', type=str, required=True)
+    parser.add_argument('--dir', help='full path to global-workflow build', type=str, required=True)
 
-    args = parser.parse_args()
-    return args
+    return parser.parse_args()
 
 
 if __name__ == '__main__':
 
     user_inputs = input_args()
-    setup_expt_args = YAMLFile(path=user_inputs.yaml)
-
     HOMEgfs = Path.absolute(Path(user_inputs.dir))
-    type = setup_expt_args.experiment.type
-    mode = setup_expt_args.experiment.mode
-
-    setup_expt_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_expt.py'))
-
-    setup_expt_cmd.add_default_arg(type)
-    setup_expt_cmd.add_default_arg(mode)
+    testconf = YAMLFile(path=user_inputs.yaml)
+    experiment_dir = Path.absolute(Path.joinpath(Path(testconf.arguments.expdir), Path(testconf.arguments.pslot)))
 
-    for conf, value in setup_expt_args.arguments.items():
-        setup_expt_cmd.add_default_arg(f'--{conf}')
-        setup_expt_cmd.add_default_arg(str(value))
+    # Create a list of arguments to setup_expt.py
+    setup_expt_args = [testconf.experiment.type, testconf.experiment.mode]  # TODO: rename 'type' as 'system' in case.yaml
+    for kk, vv in testconf.arguments.items():
+        setup_expt_args.append(f"--{kk}")
+        setup_expt_args.append(str(vv))
 
-    logger.info(f'Run command: {setup_expt_cmd.command}')
-    setup_expt_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stderr'))
-    setup_expt_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_expt.stdout'))
-    print(setup_expt_stderr)
-    setup_expt_cmd(output=setup_expt_stdout, error=setup_expt_stderr)
+    logger.info(f'Call: setup_expt.main()')
+    setup_expt.main(setup_expt_args)
 
-    setup_xml_cmd = Executable(Path.joinpath(HOMEgfs, 'workflow', 'setup_xml.py'))
-    expdir = Path.absolute(Path.joinpath(Path(setup_expt_args.arguments.expdir), Path(setup_expt_args.arguments.pslot)))
-    setup_xml_cmd.add_default_arg(str(expdir))
+    # Create a list of arguments to setup_xml.py
+    setup_xml_args = [str(experiment_dir)]
 
-    logger.info(f'Run command: {setup_xml_cmd.command}')
-    setup_xml_stderr = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stderr'))
-    setup_xml_stdout = str(Path.joinpath(HOMEgfs, 'ci', 'scripts', 'setup_xml.stdout'))
-    setup_xml_cmd(output=setup_xml_stdout, error=setup_xml_stderr)
+    logger.info(f"Call: setup_xml.main()")
+    setup_xml.main(setup_xml_args)

workflow/setup_expt.py

@@ -292,6 +292,11 @@ def edit_baseconfig(host, inputs, yaml_dict):
     extend_dict = get_template_dict(host.info)
     tmpl_dict = dict(tmpl_dict, **extend_dict)
 
+    if inputs.start in ["warm"]:
+        is_warm_start = ".true."
+    elif inputs.start in ["cold"]:
+        is_warm_start = ".false."
+
     extend_dict = dict()
     extend_dict = {
         "@PSLOT@": inputs.pslot,
@@ -300,7 +305,7 @@ def edit_baseconfig(host, inputs, yaml_dict):
         "@CASECTL@": f'C{inputs.resdet}',
         "@EXPDIR@": inputs.expdir,
         "@ROTDIR@": inputs.comrot,
-        "@EXP_WARM_START@": inputs.warm_start,
+        "@EXP_WARM_START@": is_warm_start,
         "@MODE@": inputs.mode,
         "@gfs_cyc@": inputs.gfs_cyc,
         "@APP@": inputs.app
@@ -369,7 +374,7 @@ def get_template_dict(input_dict):
     return output_dict
 
 
-def input_args():
+def input_args(*argv):
     """
     Method to collect user arguments for `setup_expt.py`
     """
@@ -451,15 +456,7 @@ def input_args():
     gefs.add_argument('--yaml', help='Defaults to substitute from', type=str, required=False,
                       default=os.path.join(_top, 'parm/config/gefs/yaml/defaults.yaml'))
 
-    args = parser.parse_args()
-
-    # Add an entry for warm_start = .true. or .false.
-    if args.start in ['warm']:
-        args.warm_start = ".true."
-    elif args.start in ['cold']:
-        args.warm_start = ".false."
-
-    return args
+    return parser.parse_args(argv[0][0] if len(argv[0]) else None)
 
 
 def query_and_clean(dirname):
@@ -493,9 +490,9 @@ def validate_user_request(host, inputs):
             raise NotImplementedError(f"Supported resolutions on {machine} are:\n{', '.join(supp_res)}")
 
 
-if __name__ == '__main__':
+def main(*argv):
 
-    user_inputs = input_args()
+    user_inputs = input_args(argv)
     host = Host()
 
     validate_user_request(host, user_inputs)
@@ -514,3 +511,8 @@ def validate_user_request(host, inputs):
         makedirs_if_missing(expdir)
         fill_EXPDIR(user_inputs)
         update_configs(host, user_inputs)
+
+
+if __name__ == '__main__':
+
+    main()

workflow/setup_xml.py

@@ -11,7 +11,7 @@
 from wxflow import Configuration
 
 
-def input_args():
+def input_args(*argv):
     """
     Method to collect user arguments for `setup_xml.py`
     """
@@ -37,9 +37,7 @@ def input_args():
     parser.add_argument('--verbosity', help='verbosity level of Rocoto', type=int,
                         default=10, required=False)
 
-    args = parser.parse_args()
-
-    return args
+    return parser.parse_args(argv[0][0] if len(argv[0]) else None)
 
 
 def check_expdir(cmd_expdir, cfg_expdir):
@@ -51,9 +49,9 @@ def check_expdir(cmd_expdir, cfg_expdir):
         raise ValueError('Abort!')
 
 
-if __name__ == '__main__':
+def main(*argv):
 
-    user_inputs = input_args()
+    user_inputs = input_args(argv)
     rocoto_param_dict = {'maxtries': user_inputs.maxtries,
                          'cyclethrottle': user_inputs.cyclethrottle,
                          'taskthrottle': user_inputs.taskthrottle,
@@ -74,3 +72,8 @@ def check_expdir(cmd_expdir, cfg_expdir):
     # Create Rocoto Tasks and Assemble them into an XML
     xml = rocoto_xml_factory.create(f'{net}_{mode}', app_config, rocoto_param_dict)
     xml.write()
+
+
+if __name__ == '__main__':
+
+    main()