Added validation for YAMLSpecication objects via jsonschema

added checks for valid keys

branch updates

working on validation

Added pre-commit to enable flake8 checks (LLNL#244)

Added pre-commit to enable flake8 checks before a commit is accepted.

Also reordered requirements.txt to more
easily determine which are for development.

Bugfix for logging that didn't appear in submodules (LLNL#247)

* Improved logging setup.

* Transition to a LoggerUtil class.

* Addition of docstring to LoggerUtility + cleanup.

added schema file

updates

fixed spec

fixed spec

added jsonschema to deps

updates

ran black on yamlspecification.py

specified newest jsonschema version

added manifest

added include_package_data to setup.py

reformatted json

experimental package_data

fixed path

fixed path

fixed path again

reverted newline

added check for empty strings

reworked exception logic

implemented reviewer suggestions, shifted exception logic, renamed redundant variables

renamed variable

removed unused import

added missing `self.verify_environment()` call

Co-Authored-By: Francesco Di Natale <frank.dinatale1988@gmail.com>

paths and git dependencies are now array types

Co-Authored-By: Francesco Di Natale <frank.dinatale1988@gmail.com>

removed redundant logic

swapped number type to integer

moved env schema validation to top, which avoids some types of ambiguous errors

removed test yaml

removed some additionalProperties restrictions

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+# See https://pre-commit.com for more information
+# See https://pre-commit.com/hooks.html for more hooks
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.5.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: check-toml
+    -   id: flake8

MANIFEST.in

@@ -0,0 +1 @@
+include maestrowf/datastructures/schemas.json

Pipfile

@@ -4,23 +4,28 @@ verify_ssl = true
 name = "pypi"
 
 [packages]
-"enum34" = "*"
-filelock = "*"
+PyYAML = ">=4.2b1"
 six = "*"
+filelock = "*"
 tabulate = "*"
-Fabric = "*"
-PyYAML = ">= 4.2b1"
+dill = "*"
 maestrowf = {path = "."}
 
 [dev-packages]
-"flake8" = "*"
+flake8 = "*"
 pydocstyle = "*"
 pylint = "*"
 tox = "*"
 coverage = "*"
-sphinx_rtd_theme = "*"
-Sphinx = "*"
 pytest = "*"
+fabric = "*"
+Sphinx = "*"
+pytest-cov = "*"
+pre-commit = "*"
+sphinx-rtd-theme = "*"
+
+[pipenv]
+allow_prereleases = true
 
 [requires]
 python_version = "3.6"

maestrowf/datastructures/schemas.json

@@ -0,0 +1,115 @@
+{
+  "DESCRIPTION": {
+    "type": "object",
+    "properties": {
+      "name": {"type": "string", "minLength": 1},
+      "description": {"type": "string", "minLength": 1}
+    },
+    "required": [
+      "name",
+      "description"
+    ]
+  },
+  "PARAM": {
+    "type": "object",
+    "properties": {
+      "values": {
+        "type": "array"
+      },
+      "label": {"type": "string", "minLength": 1}
+    },
+    "additionalProperties": false,
+    "required": [
+      "values",
+      "label"
+    ]
+  },
+  "STUDY_STEP": {
+    "type": "object",
+    "properties": {
+      "name": {"type": "string", "minLength": 1},
+      "description": {"type": "string", "minLength": 1},
+      "run": {
+        "type": "object",
+        "properties": {
+          "cmd": {"type": "string", "minLength": 1},
+          "depends": {"type": "array", "uniqueItems": true},
+          "pre": {"type": "string", "minLength": 1},
+          "post": {"type": "string", "minLength": 1},
+          "restart": {"type": "string", "minLength": 1},
+          "nodes": {"type": "integer"},
+          "procs": {"type": "integer"},
+          "gpus": {"type": "integer"},
+          "cores per task": {"type": "integer"},
+          "walltime": {"type": "string", "minLength": 1},
+          "reservation": {"type": "string", "minLength": 1}
+        },
+        "additionalProperties": false,
+        "required": [
+          "cmd"
+        ]
+      }
+    },
+    "additionalProperties": false,
+    "required": [
+      "name",
+      "description",
+      "run"
+    ]
+  },
+  "ENV": {
+    "type": "object",
+    "properties": {
+      "variables": {"type": "object"},
+      "labels": {"type": "object"},
+      "sources": {"type": "object"},
+      "dependencies": {
+        "type": "object",
+        "properties": {
+          "paths": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "name": {"type": "string", "minLength": 1},
+                "path": {"type": "string", "minLength": 1}
+              },
+              "required": [
+                "name",
+                "path"
+              ],
+              "additionalProperties": false
+            }
+          },
+          "git": {
+            "type": "array",
+            "items": {
+              "properties": {
+                "name": {"type": "string", "minLength": 1},
+                "path": {"type": "string", "minLength": 1},
+                "url": {"type": "string", "minLength": 1}
+              },
+              "required": [
+                "name",
+                "path",
+                "url"
+              ]
+            }
+          },
+          "spack": {
+            "type": "object",
+            "properties": {
+              "name": {"type": "string", "minLength": 1},
+              "package_name": {"type": "string", "minLength": 1}
+            },
+            "required": [
+              "type",
+              "package_name"
+            ]
+          }
+        }
+      }
+    },
+    "additionalProperties": false
+  }
+}