Support (de)serializing yaml-format recipes

requirements_dev.txt

@@ -2,6 +2,7 @@ bidict==0.21.4
 pytest==7.1.1
 pytest-cov==3.0.0
 pytest-timeout==2.1.0
+ruamel.yaml
 setuptools==60.10.0
 stomp.py==8.0.0
 pika==1.2.0

setup.cfg

@@ -28,6 +28,7 @@ project_urls =
 install_requires =
     bidict
     pika
+    ruamel.yaml
     setuptools
     stomp.py>=7
 packages = find:

src/workflows/recipe/recipe.py

@@ -1,15 +1,33 @@
 from __future__ import annotations
 
 import copy
+import io
 import json
 import string
 from typing import Any, Dict
 
+from ruamel.yaml import YAML
+from ruamel.yaml.constructor import SafeConstructor
+
 import workflows
 
 basestring = (str, bytes)
 
 
+def construct_yaml_map(self, node):
+    # Test if there are duplicate node keys
+    # In the case of duplicate keys, last wins
+    data = {}
+    yield data
+    for key_node, value_node in node.value:
+        key = self.construct_object(key_node, deep=True)
+        val = self.construct_object(value_node, deep=True)
+        data[key] = val
+
+
+SafeConstructor.add_constructor("tag:yaml.org,2002:map", construct_yaml_map)
+
+
 class Recipe:
     """Object containing a processing recipe that can be passed to services.
     A recipe describes how all involved services are connected together, how
@@ -29,7 +47,9 @@ def __init__(self, recipe=None):
     def deserialize(self, string):
         """Convert a recipe that has been stored as serialized json string to a
         data structure."""
-        return self._sanitize(json.loads(string))
+        with YAML(typ="safe") as yaml:
+            yaml.allow_duplicate_keys = True
+            return self._sanitize(yaml.load(string))
 
     @staticmethod
     def _sanitize(recipe):
@@ -50,9 +70,20 @@ def _sanitize(recipe):
             recipe["start"] = [tuple(x) for x in recipe["start"]]
         return recipe
 
-    def serialize(self):
-        """Write out the current recipe as serialized json string."""
-        return json.dumps(self.recipe)
+    def serialize(self, format: str = "json"):
+        """Write out the current recipe as serialized string.
+
+        Supported serialization formats are "json" and "yaml".
+        """
+        if format == "json":
+            return json.dumps(self.recipe)
+        elif format == "yaml":
+            buf = io.StringIO()
+            with YAML(output=buf, typ="safe") as yaml:
+                yaml.dump(self.recipe)
+            return buf.getvalue()
+        else:
+            raise ValueError(f"Unsupported serialization format {format}")
 
     def pretty(self):
         """Write out the current recipe as serialized json string with pretty formatting."""

tests/recipe/test_recipe.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import json
 from unittest import mock
 
 import pytest
+from ruamel.yaml import YAML
 
 import workflows
 import workflows.recipe
@@ -100,6 +102,9 @@ def test_serializing_and_deserializing_recipes():
     assert A.deserialize(A.serialize()) == A.recipe
     assert B.deserialize(B.serialize()) == B.recipe
 
+    assert A.deserialize(A.serialize(format="yaml")) == A.recipe
+    assert B.deserialize(B.serialize(format="yaml")) == B.recipe
+
 
 def test_validate_tests_for_empty_recipe():
     """Validating a recipe that has not been defined must throw an error."""
@@ -368,3 +373,120 @@ def test_merging_recipes():
             C.recipe.values(),
         )
     )
+
+
+def test_deserialize_json():
+    healthy_recipe = """
+        {
+          "1": {
+            "queue": "my.queue",
+            "parameters": {
+              "foo": "bar",
+              "ingredients": ["ham", "spam"],
+              "workingdir": "/path/to/workingdir",
+              "output_file": "out.txt"
+            }
+          },
+          "start": [
+            [1, []]
+          ]
+        }
+    """
+    A = workflows.recipe.Recipe(healthy_recipe)
+    assert A.recipe == {
+        "start": [(1, [])],
+        1: {
+            "queue": "my.queue",
+            "parameters": {
+                "foo": "bar",
+                "ingredients": ["ham", "spam"],
+                "workingdir": "/path/to/workingdir",
+                "output_file": "out.txt",
+            },
+        },
+    }
+
+
+def test_deserialize_yaml():
+    healthy_recipe = """
+1:
+  queue: my.queue
+  parameters:
+    foo: bar
+    ingredients:
+    - ham
+    - spam
+    workingdir: /path/to/workingdir
+    output_file: out.txt
+start:
+- [1, []]
+"""
+    A = workflows.recipe.Recipe(healthy_recipe)
+    assert A.recipe == {
+        "start": [(1, [])],
+        1: {
+            "queue": "my.queue",
+            "parameters": {
+                "foo": "bar",
+                "ingredients": ["ham", "spam"],
+                "workingdir": "/path/to/workingdir",
+                "output_file": "out.txt",
+            },
+        },
+    }
+
+
+def test_serialize_json():
+    A, B = generate_recipes()
+
+    assert (
+        A.recipe
+        == workflows.recipe.Recipe(json.loads(A.serialize(format="json"))).recipe
+    )
+    assert (
+        B.recipe
+        == workflows.recipe.Recipe(json.loads(B.serialize(format="json"))).recipe
+    )
+
+
+def test_serialize_yaml():
+    A, B = generate_recipes()
+
+    # Verify that this definitely isn't json
+    with pytest.raises(json.JSONDecodeError):
+        json.loads(A.serialize(format="yaml"))
+
+    with YAML(typ="safe") as yaml:
+        assert (
+            A.recipe
+            == workflows.recipe.Recipe(yaml.load(A.serialize(format="yaml"))).recipe
+        )
+        assert (
+            B.recipe
+            == workflows.recipe.Recipe(yaml.load(B.serialize(format="yaml"))).recipe
+        )
+
+
+def test_unsupported_serialization_format_raises_error():
+    A, _ = generate_recipes()
+
+    with pytest.raises(ValueError):
+        A.serialize(format="xml")
+
+
+def test_json_recipe_with_pseudo_comment():
+    recipe = """
+        {
+          "1": ["This is a comment"],
+          "1": {
+            "parameters": {
+              "foo": "bar"
+            }
+          },
+          "start": [
+            [1, []]
+          ]
+        }
+    """
+    A = workflows.recipe.Recipe(recipe)
+    assert A.recipe == {"start": [(1, [])], 1: {"parameters": {"foo": "bar"}}}

tests/recipe/test_validate.py

@@ -4,11 +4,11 @@
 
 from __future__ import annotations
 
-import json
 import sys
 from unittest import mock
 
 import pytest
+import ruamel.yaml
 
 import workflows
 from workflows.recipe.validate import main, validate_recipe
@@ -64,8 +64,8 @@ def test_value_error_when_validating_bad_json(tmpdir):
     recipe_file = tmpdir.join("recipe.json")
     recipe_file.write(bad_json)
 
-    # Run validate with mock open, expect JSON error
-    with pytest.raises(json.JSONDecodeError):
+    # Run validate with mock open, expect parsing error
+    with pytest.raises(ruamel.yaml.parser.ParserError):
         validate_recipe(recipe_file.strpath)
 
 

tests/recipe/test_wrapped_recipe.py

@@ -4,7 +4,7 @@
 
 import pytest
 
-import workflows
+import workflows.transport.common_transport
 from workflows.recipe import Recipe
 from workflows.recipe.wrapper import RecipeWrapper