From dfef6445546abbefce32e3b2f718b9e77a90f51e Mon Sep 17 00:00:00 2001 From: Francesco Di Natale Date: Thu, 5 Dec 2019 11:39:55 -0800 Subject: [PATCH] Release of 1.1.5 to mainline master (#206) * Addition of user enabled workspace hashing (#145) * Addition of hashing to Study parameterization. * Addition of the hashws option to argparse. * Addition of a warning note for users who use labels in steps. * Update setup.py to 1.1.4dev * More generalized FluxScriptAdapter (#149) * Addition of a more general flux ScriptAdapter. * Addition of some casting from int to str * Corrected "gpus" to "ngpus" * Rework jobspec construction to make a valid jobspec. * Check for empty value for cores per task. * README tweak to update quickstart link. (#139) * typos. fixes #141 (#154) * Correction of flake8 style errors [new version of flake8]. * Update to setup.py to reflect dev version 1.0 * Correction to safe pathing for missed cases and make_safe_path enhancements. (#157) * Made pickle and log path string safe for pathing. * Tweaks to make_safe_path to include a base path. * Updates to make_safe_path usage * Correction to not modify the iterator copy. * Correction to fix the format of output status time to avoid a comma that breaks printing. (#160) * Addition of a utility function for formatting times to H:M:S * _StepRecord time methods now call the new utility function. * Tweaks to add days to the format to avoid 3 digit hours. * Tweak to formatting. * Made the day format more parsable. * Removal of _stage_linear since it is now not needed. (#156) * Removal of _stage_linear since it is now not needed. * Addition of linear LULESH samples. * Update the dev to 1.1. * Addition of pargs for passing parameters to custom parameter generation (#152) * Addition of a utility method to create a dictionary from a list of key-value pairs. * Addition of the pargs interface for passing parameters to custom parameter generation. * Addition of a Monte Carlo example that accepts pargs. * Addition of pargs check for dependency on pgen. * Addition of clearer error message for malformed parameters. * Update setup.py * do not overwrite log file. (#162) Signed-off-by: Peter Robinson * Added confirmation message after launching a study (#163) * Enhancements to store relative pathing in the metadata file. (#165) * Changes to make workspaces reflect relative pathing based on step names. * Addition of an alternative output format based on step combinations. * Addition of tag to LULESH git dependency. (#169) * Script Adapter Plugin (#167) (#170) Fixes #167 * added pytest to requirements added Pipfile and pipenv settings * Added property key to Abstract.ScriptAdapter (#167) Also added impementation and tests to verify that existing functionality isn't changed * updated factory to use key when registering adapters(#167) * cleanedup linelength * cleaned up imports to be specific to module (#167) * added tests to verify exception for unknown adapter * moved adapters tests to individual files * added test to verify scriptadapter functionality (#167) updated gitignore to have testing and pycharm ignores testing existing adapters in factory (#167) added test to verify factories.keys matches get_valid_adapters (#167) added copyright to file * updated __init__ modules to do dynamic includes * removed unneeeded imports * updated dependency versions * fixed all flake8 errors * updated to run flake8 and pytest when run locally * updated tests to have documentation about purpose and function as requested in #170 * fixed line length * Removal of nose from requirements. * updated to remove nose from the requirements * PyYAML vulnerability fix (#171) * Locking the version of PyYAML to be above 2.1 because of an arbitrary code execution vulnerability. * Addition of a version condition to pyyaml to patch a vulnerability. * Update of Pipfile.lock to match Pipefile. * Minor tweak to indentation for flake8 failure. * fixed pyyaml to requirements (#172) * Addition of a loader to the yaml load call. (#174) Fixes #173 * Addition of a loader to the yaml load call. * Addition of a catch if the loader attribute is missing. * Correction to install enum34 for Python versions < 3.4 (#176) * Moved enum34 to condition dependent on Python<3.4. * Addition of conditional enum34 install for requirements.txt. * Correction of requirements.txt syntax for python version. * Addition of a Dockerfile for tutorials and ease of trying out. (#178) * Addition of a Dockerfile for quick tutorials. * Tweaks for Docker and addition of git. * Tweak to Docker file for caching. * Addition of Docker documentation. * Tweaks to Docker documentation. * Removal of markdown ## * Take out shebang from shell definition and add it when script is written. (#181) * Take out shebang from shell definiton and at it when script is written. * Include shebang in cmd and fix format of string written to file. * Tweaks to fix malformed log statements. (#182) * Correction to message when stating no to launch. * Enhance shell batch setting to apply to scheduler scripts. (#183) * Extension of shebang feature to allow users to specify shells. * Addition of debug message to print kwargs. * Addition of kwargs. * Addition of basic batch settings to LULESH sample. * Addition of kwargs to Flux adapters. * Docstring tweaks. * Docstring update. * Fixes the addition of the shebang header for SLURM (#184) * Docstring correction for LocalAdapter. * Correction to addition of exec line at top of scripts. * Correction to an accidental reassignment of cmd. * Removal of an assignment of self._exec in SLURM adapter. * Change to transition adapter returns to Record objects. (#177) * Addition of a Record class for storing general data. * Addition of SubmissionRecord type. * Update to the order of for record parameters. * Changes to StepRecord to expect SubmissionRecord returns. * Updates to SLURM and local adapters to use SubmissionRecords. * Slight tweak to LocalAdapter docstring. * Tweak to have SubmissionRecord initialize its base. * Addition of CancellationRecord class. * Changes to CancellationRecord to map based on status. * Additional interface additions and tweaks. * Changes to have cancel use CancellationRecords. * Update to ExecutionGraph to use records. * Updates to SLURM and local adapters to use SubmissionRecords. * Slight tweak to LocalAdapter docstring. * Addition of CancellationRecord class. * Additional interface additions and tweaks. * Changes to have cancel use CancellationRecords. * Cherry pick of execution commit. * Removal of redundant "get" definiton. * Addition of a SLURM enabled LULESH sample specification. * Addition of output for stdout and stderr for Local adapter. * Correction of file to open. * Addition of 3.7 to testing stack. * Added 3.7 to tox.ini. * Removal of py37 in testing. * Addition of py37 to travisCI (#187) * Correction to 3.7 specification. * Removed py37 from tox testing. * Readded py37 to tox and removed duplicate from travis. * Addition of build status badge. * Update SLURM sample spec to add missing walltime. * Addition of dumping the environment to a YAML file. (#190) * Addition of documentation that covers the set up of a simple study (#168) * Addition of simple Hello World spec. * Addition of basics page to index. * Addition of hello_world documentation. * Additions to hello_world. * More documentation in single step section. * Continued edits to Hello World. * Addition of parameter section. * Addition of a note about %% token. * Addition of directory structure. * Continuation of parameter documentation. * Removal of the depends key. * Addition of the env section description. * Addition of a link to Docker documentation for Dockerfiles. * Addition of single parameter hello world. * Correction of double colons. * Correction of indentation. * Addition of print out to verify output. * Addition of sample specifications for multi and single params. * Addition of more documentation for single param. * Additional output to show parameter results. * Correction to formatting. * Addition of samples. * Addition of simple Hello World spec. * Addition of basics page to index. * Addition of hello_world documentation. * Additions to hello_world. * More documentation in single step section. * Continued edits to Hello World. * Addition of parameter section. * Addition of a note about %% token. * Addition of directory structure. * Continuation of parameter documentation. * Removal of the depends key. * Addition of the env section description. * Addition of a link to Docker documentation for Dockerfiles. * Addition of single parameter hello world. * Correction of double colons. * Correction of indentation. * Addition of print out to verify output. * Addition of sample specifications for multi and single params. * Addition of more documentation for single param. * Additional output to show parameter results. * Correction to formatting. * Updates to docstrings for data structures. * Updates to clear Sphinx warnings. * Removal of escape on the *args becuase of flake8 failure. * Clean up of existing hello world specs. * Addition of multistep example spec. * Removal of * to fix sphinx errors. * Correction to some docstrings. * Tweaks to specs for consistent naming. * Finished multi-step parameterized example. * Tweaks to hello world docs. * Addition of link to examples on GitHub. * Correction of link to examples. * Correction of link to examples (again). * Removal of Pipfile.lock. * Additions to gitignore for vscode and pipenv. * Marking for v1.1.4 release. * Corrected a missed merge for release v1.1.4 * Extend the Specification interface to break out loading from streams. (#198) * Closes #198 * Addition of loading specification "from_str". * Updates to Specification docstrings. * Updates to abstract Specification to change from str to stream. * Updates to YAMLSpecification to use the new stream API. * Removal of IOString * Update to the YAMLSpecification load stream method. * Quickfix: Addition of the accidental removal of the path member variable. * Updating the version to 1.1.5dev (forgotten previously). * Correction to versioning for install. * Moved SLURM parameters to be placed in front of the submitted script. (#202) Fixes #201 * Addition of version information to package and command line (#205) * Addition of version information. * Tweak to have setup.py pull from __version__ * Addition of command line arg to print version. * Pinning version for release 1.1.5 --- maestrowf/__init__.py | 3 ++ maestrowf/abstracts/specification.py | 15 ++++++-- maestrowf/datastructures/yamlspecification.py | 34 ++++++++++++++----- .../interfaces/script/slurmscriptadapter.py | 2 +- maestrowf/maestro.py | 3 ++ setup.py | 10 +++--- 6 files changed, 50 insertions(+), 17 deletions(-) diff --git a/maestrowf/__init__.py b/maestrowf/__init__.py index 789d2fe67..195ac7f49 100644 --- a/maestrowf/__init__.py +++ b/maestrowf/__init__.py @@ -50,3 +50,6 @@ def emit(self, record): LOGGER = logging.getLogger(__name__) LOGGER.addHandler(NullHandler()) + +__version_info__ = ("1", "1", "5") +__version__ = '.'.join(__version_info__) diff --git a/maestrowf/abstracts/specification.py b/maestrowf/abstracts/specification.py index 546e7a365..a5b6ae43a 100644 --- a/maestrowf/abstracts/specification.py +++ b/maestrowf/abstracts/specification.py @@ -42,10 +42,21 @@ class Specification(object): @abstractclassmethod def load_specification(cls, path): """ - Method for loading a study specification. + Method for loading a study specification from a file. :param path: Path to a study specification. - :returns: A specification object containing the information from path. + :returns: A specification object containing the information loaded + from path. + """ + pass + + @abstractclassmethod + def load_specification_from_stream(cls, stream): + """ + Method for loading a study specification from a stream. + + :param stream: Raw text stream containing specification data. + :returns: A specification object containing the information in string. """ pass diff --git a/maestrowf/datastructures/yamlspecification.py b/maestrowf/datastructures/yamlspecification.py index 262e6b040..c66d2e6c3 100644 --- a/maestrowf/datastructures/yamlspecification.py +++ b/maestrowf/datastructures/yamlspecification.py @@ -95,22 +95,38 @@ def load_specification(cls, path): try: # Load the YAML spec from the file. with open(path, 'r') as data: - try: - spec = yaml.load(data, yaml.FullLoader) - except AttributeError: - logger.warning( - "*** PyYAML is using an unsafe version with a known " - "load vulnerability. Please upgrade your installation " - "to a more recent version! ***") - spec = yaml.load(data) + specification = cls.load_specification_from_stream(data) except Exception as e: logger.exception(e.args) raise + # Populate the path to the specification that populated this instance. + specification.path = path + return specification + + @classmethod + def load_specification_from_stream(cls, stream): + """ + Load a study specification. + + :param stream: Raw text stream to study YAML specification data. + :returns: A specification object containing the information from the + passed stream. + """ + + try: + spec = yaml.load(stream, yaml.FullLoader) + except AttributeError: + logger.warning( + "*** PyYAML is using an unsafe version with a known " + "load vulnerability. Please upgrade your installation " + "to a more recent version! ***") + spec = yaml.load(stream) + logger.debug("Loaded specification -- \n%s", spec["description"]) specification = cls() - specification.path = path + specification.path = None specification.description = spec.pop("description", {}) specification.environment = spec.pop("env", {'variables': {}, diff --git a/maestrowf/interfaces/script/slurmscriptadapter.py b/maestrowf/interfaces/script/slurmscriptadapter.py index 9f2077a0f..0643e9db8 100644 --- a/maestrowf/interfaces/script/slurmscriptadapter.py +++ b/maestrowf/interfaces/script/slurmscriptadapter.py @@ -179,7 +179,7 @@ def submit(self, step, path, cwd, job_map=None, env=None): cmd += ["--reservation", self._batch["reservation"]] # Append the script path and working directory. - cmd += [path, "-D", cwd] + cmd += ["-D", cwd, path] cmd = " ".join(cmd) LOGGER.debug("cwd = %s", cwd) diff --git a/maestrowf/maestro.py b/maestrowf/maestro.py index b4624b1e6..9c29c6559 100644 --- a/maestrowf/maestro.py +++ b/maestrowf/maestro.py @@ -39,6 +39,7 @@ import tabulate import time +from maestrowf import __version__ from maestrowf.conductor import monitor_study from maestrowf.datastructures import YAMLSpecification from maestrowf.datastructures.core import Study @@ -392,6 +393,8 @@ def setup_argparser(): parser.add_argument( "-c", "--logstdout", action="store_true", default=True, help="Log to stdout in addition to a file. [Default: %(default)s]") + parser.add_argument( + "-v", "--version", action="version", version='%(prog)s ' + __version__) return parser diff --git a/setup.py b/setup.py index b5c197966..d4099b7a3 100644 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ +from maestrowf import __version__ from setuptools import setup, find_packages setup(name='maestrowf', description='A tool and library for specifying and conducting general ' 'workflows.', - version='1.1.4', + version=__version__, author='Francesco Di Natale', author_email='dinatale3@llnl.gov', url='https://github.com/llnl/maestrowf', @@ -20,18 +21,17 @@ 'six', "filelock", "tabulate", + "enum34 ; python_version<'3.4'" ], - extras_require={ - ":python_version<'3.4'": ['enum34'], - }, + extras_require={}, classifiers=[ 'Development Status :: 3 - Alpha', 'Intended Audience :: Developers', 'Operating System :: Unix', 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', ], )