workflow RDF

README.md

@@ -147,10 +147,14 @@ As a dependency it is included in [bioimageio.core](https://github.com/bioimage-
 | BIOIMAGEIO_CACHE_WARNINGS_LIMIT | "3" | Maximum number of warnings generated for simple cache hits. |
 
 ## Changelog
-#### bioimageio.spec 0.4.9
-- small bugixes
-- better type hints
-- improved tests 
+#### bioimageio.spec tbd
+- make pre-/postprocessing kwargs `mode` and `axes` always optional for model RDF 0.3 and 0.4
+- rename 
+  - `ImportableSource`→`CallableSource`
+  - `ImportableModule`→`CallableFromModule`
+  - `ImportableSourceFile`→`CallableFromSourceFile`
+  - `ResolvedImportableSourceFile`→`ResolvedCallableFromSourceFile`
+  - `LocalImportableModule`→`LocalCallableFromModule`
 
 #### bioimageio.spec 0.4.8post1
 - add `axes` and `eps` to `scale_mean_var`

bioimageio/spec/__init__.py

@@ -1,4 +1,4 @@
-from . import collection, model, rdf, shared
+from . import collection, model, rdf, shared, workflow
 from .commands import update_format, update_rdf, validate
 from .io_ import (
     get_resource_package_content,

bioimageio/spec/__main__.py

@@ -15,7 +15,9 @@
     enrich_partial_rdf_with_imjoy_plugin = None
     partner_help = ""
 else:
-    partner_help = f"\n+\nbioimageio.spec.partner {__version__}\nimplementing:\n\tpartner collection RDF {collection.format_version}"
+    partner_help = (
+        f"\nbioimageio.spec.partner {__version__}\nimplementing:\n\tpartner collection RDF {collection.format_version}"
+    )
 
 help_version = (
     f"bioimageio.spec {__version__}"

bioimageio/spec/model/v0_3/raw_nodes.py

@@ -11,8 +11,8 @@
 from bioimageio.spec.shared.raw_nodes import (
     Dependencies,
     ImplicitOutputShape,
-    ImportableModule,
-    ImportableSourceFile,
+    CallableFromModule,
+    CallableFromSourceFile,
     ParametrizedInputShape,
     RawNode,
     URI,
@@ -136,7 +136,7 @@ class TensorflowSavedModelBundleWeightsEntry(_WeightsEntryBase):
     TensorflowSavedModelBundleWeightsEntry,
 ]
 
-ImportableSource = Union[ImportableSourceFile, ImportableModule]
+CallableSource = Union[CallableFromSourceFile, CallableFromModule]
 
 
 @dataclass
@@ -168,7 +168,7 @@ class Model(RDF_Base):
     timestamp: datetime = missing
     type: Literal["model"] = missing
 
-    source: Union[_Missing, ImportableSource] = missing
+    source: Union[_Missing, CallableSource] = missing
     test_inputs: List[Union[URI, Path]] = missing
     test_outputs: List[Union[URI, Path]] = missing
     weights: Dict[WeightsFormat, WeightsEntry] = missing

bioimageio/spec/model/v0_3/schema.py

@@ -602,7 +602,7 @@ class Meta:
         + " This field is only required if the field source is present.",
     )
 
-    source = fields.ImportableSource(
+    source = fields.CallableSource(
         bioimageio_maybe_required=True,
         bioimageio_description="Language and framework specific implementation. As some weights contain the model "
         "architecture, the source is optional depending on the present weight formats. `source` can either point to a "

bioimageio/spec/model/v0_4/raw_nodes.py

@@ -26,8 +26,8 @@
 from bioimageio.spec.shared.raw_nodes import (
     Dependencies,
     ImplicitOutputShape,
-    ImportableModule,
-    ImportableSourceFile,
+    CallableFromModule,
+    CallableFromSourceFile,
     ParametrizedInputShape,
     RawNode,
     URI,
@@ -56,7 +56,7 @@
     "pytorch_state_dict", "torchscript", "keras_hdf5", "tensorflow_js", "tensorflow_saved_model_bundle", "onnx"
 ]
 
-ImportableSource = Union[ImportableSourceFile, ImportableModule]
+CallableSource = Union[CallableFromSourceFile, CallableFromModule]
 
 
 @dataclass
@@ -77,7 +77,7 @@ class OnnxWeightsEntry(_WeightsEntryBase, OnnxWeightsEntry03):
 @dataclass
 class PytorchStateDictWeightsEntry(_WeightsEntryBase):
     weights_format_name = "Pytorch State Dict"
-    architecture: ImportableSource = missing
+    architecture: CallableSource = missing
     architecture_sha256: Union[_Missing, str] = missing
     kwargs: Union[_Missing, Dict[str, Any]] = missing
     pytorch_version: Union[_Missing, packaging.version.Version] = missing

bioimageio/spec/model/v0_4/schema.py

@@ -204,7 +204,7 @@ class OnnxWeightsEntry(OnnxWeightsEntry03, _WeightsEntryBase):
 class PytorchStateDictWeightsEntry(_WeightsEntryBase):
     bioimageio_description = "PyTorch state dictionary weights format"
     weights_format = fields.String(validate=field_validators.Equal("pytorch_state_dict"), required=True, load_only=True)
-    architecture = fields.ImportableSource(
+    architecture = fields.CallableSource(
         required=True,
         bioimageio_description="Source code of the model architecture that either points to a "
         "local implementation: `<relative path to file>:<identifier of implementation within the file>` or the "
@@ -226,9 +226,9 @@ class PytorchStateDictWeightsEntry(_WeightsEntryBase):
     @validates_schema
     def sha_for_source_code_file(self, data, **kwargs):
         arch = data.get("architecture")
-        if isinstance(arch, raw_nodes.ImportableModule):
+        if isinstance(arch, raw_nodes.CallableFromModule):
             return
-        elif isinstance(arch, raw_nodes.ImportableSourceFile):
+        elif isinstance(arch, raw_nodes.CallableFromSourceFile):
             sha = data.get("architecture_sha256")
             if sha is None:
                 raise ValidationError(

bioimageio/spec/rdf/v0_2/raw_nodes.py

@@ -165,6 +165,9 @@ def __post_init__(self):
         if self.type is missing:
             self.type = self.__class__.__name__.lower()
 
+        if self.format_version is missing:
+            self.format_version = get_args(FormatVersion)[-1]
+
         super().__post_init__()
 
 

bioimageio/spec/shared/_resolve_source.py

@@ -294,25 +294,25 @@ def _resolve_source_path(
 
 
 @resolve_source.register
-def _resolve_source_resolved_importable_path(
-    source: raw_nodes.ResolvedImportableSourceFile,
+def _resolve_source_resolved_callable_path(
+    source: raw_nodes.ResolvedCallableFromSourceFile,
     root_path: typing.Union[os.PathLike, URI] = pathlib.Path(),
     output: typing.Optional[os.PathLike] = None,
     pbar=None,
-) -> raw_nodes.ResolvedImportableSourceFile:
-    return raw_nodes.ResolvedImportableSourceFile(
+) -> raw_nodes.ResolvedCallableFromSourceFile:
+    return raw_nodes.ResolvedCallableFromSourceFile(
         callable_name=source.callable_name, source_file=resolve_source(source.source_file, root_path, output, pbar)
     )
 
 
 @resolve_source.register
-def _resolve_source_importable_path(
-    source: raw_nodes.ImportableSourceFile,
+def _resolve_source_callable_path(
+    source: raw_nodes.CallableFromSourceFile,
     root_path: typing.Union[os.PathLike, URI] = pathlib.Path(),
     output: typing.Optional[os.PathLike] = None,
     pbar=None,
-) -> raw_nodes.ResolvedImportableSourceFile:
-    return raw_nodes.ResolvedImportableSourceFile(
+) -> raw_nodes.ResolvedCallableFromSourceFile:
+    return raw_nodes.ResolvedCallableFromSourceFile(
         callable_name=source.callable_name, source_file=resolve_source(source.source_file, root_path, output, pbar)
     )
 

bioimageio/spec/shared/common.py

@@ -167,7 +167,7 @@ def get_patched_format_version(type_: str, format_version: str):
 
 
 def get_spec_type_from_type(type_: Optional[str]):
-    if type_ in ("model", "collection", "dataset"):
+    if type_ in ("model", "collection", "dataset", "workflow"):
         return type_
     else:
         return "rdf"
@@ -214,3 +214,7 @@ def nested_default_dict_as_nested_dict(nested_dd):
         return [nested_default_dict_as_nested_dict(value) for value in nested_dd]
     else:
         return nested_dd
+
+
+AXIS_LETTER_TO_NAME = dict(b="batch", t="time", c="channel", i="index")
+AXIS_NAME_TO_LETTER = {v: k for k, v in AXIS_LETTER_TO_NAME.items()}

bioimageio/spec/shared/field_validators.py

@@ -4,6 +4,7 @@
     ContainsNoneOf,
     Equal,
     Length,
+    NoneOf,
     OneOf,
     Predicate as MarshmallowPredicate,
     Range,
@@ -15,6 +16,7 @@
 ContainsNoneOf = ContainsNoneOf
 Equal = Equal
 Length = Length
+NoneOf = NoneOf
 OneOf = OneOf
 Range = Range
 

bioimageio/spec/shared/fields.py

@@ -88,6 +88,10 @@ def deserialize(
             return value
 
 
+class Boolean(DocumentedField, marshmallow_fields.Boolean):
+    pass
+
+
 class DateTime(DocumentedField, marshmallow_fields.DateTime):
     """
     Parses datetime in ISO8601 or if value already has datetime.datetime type
@@ -181,6 +185,12 @@ def __init__(self, instance: DocumentedField, *super_args, **super_kwargs):
         super().__init__(instance, *super_args, **super_kwargs)
         self.type_name += f"\\[{self.inner.type_name}\\]"  # add type of list elements
 
+    def _serialize(self, value, attr, obj, **kwargs) -> typing.Optional[typing.List[typing.Any]]:
+        if isinstance(value, str):
+            raise TypeError("Avoiding bugs by prohibiting to serialize a list from a string.")
+
+        return super()._serialize(value, attr, obj, **kwargs)
+
 
 class Number(DocumentedField, marshmallow_fields.Number):
     pass
@@ -325,15 +335,11 @@ def __init__(self, **super_kwargs):
         super().__init__(Integer(), **super_kwargs)
 
 
-class ImportableSource(String):
+class CallableFromModule(String):
     @staticmethod
     def _is_import(path):
         return ":" not in path
 
-    @staticmethod
-    def _is_filepath(path):
-        return ":" in path
-
     def _deserialize(self, *args, **kwargs) -> typing.Any:
         source_str: str = super()._deserialize(*args, **kwargs)
         if self._is_import(source_str):
@@ -343,18 +349,34 @@ def _deserialize(self, *args, **kwargs) -> typing.Any:
             object_name = source_str[last_dot_idx + 1 :]
 
             if not module_name:
-                raise ValidationError(
-                    f"Missing module name in importable source: {source_str}. Is it just missing a dot?"
-                )
+                raise ValidationError(f"Missing module name in callable source: {source_str}.")
 
             if not object_name:
                 raise ValidationError(
-                    f"Missing object/callable name in importable source: {source_str}. Is it just missing a dot?"
+                    f"Missing object/callable name in callable source: {source_str}. Is it just missing a dot?"
                 )
 
-            return raw_nodes.ImportableModule(callable_name=object_name, module_name=module_name)
+            return raw_nodes.CallableFromModule(callable_name=object_name, module_name=module_name)
+        else:
+            raise ValidationError(source_str)
 
-        elif self._is_filepath(source_str):
+    def _serialize(self, value, attr, obj, **kwargs) -> typing.Optional[str]:
+        if value is None:
+            return None
+        elif isinstance(value, raw_nodes.CallableFromModule):
+            return f"{value.module_name}.{value.callable_name}"
+        else:
+            raise TypeError(f"{value} has unexpected type {type(value)}")
+
+
+class CallableFromSourceFile(String):
+    @staticmethod
+    def _is_filepath(path):
+        return ":" in path
+
+    def _deserialize(self, *args, **kwargs) -> typing.Any:
+        source_str: str = super()._deserialize(*args, **kwargs)
+        if self._is_filepath(source_str):
             *module_uri_parts, object_name = source_str.split(":")
             module_uri = ":".join(module_uri_parts).strip(":")
 
@@ -371,7 +393,7 @@ def _deserialize(self, *args, **kwargs) -> typing.Any:
                     ),
                 ]
             )
-            return raw_nodes.ImportableSourceFile(
+            return raw_nodes.CallableFromSourceFile(
                 callable_name=object_name, source_file=source_file_field.deserialize(module_uri)
             )
         else:
@@ -380,14 +402,17 @@ def _deserialize(self, *args, **kwargs) -> typing.Any:
     def _serialize(self, value, attr, obj, **kwargs) -> typing.Optional[str]:
         if value is None:
             return None
-        elif isinstance(value, raw_nodes.ImportableModule):
-            return f"{value.module_name}.{value.callable_name}"
-        elif isinstance(value, raw_nodes.ImportableSourceFile):
+        elif isinstance(value, raw_nodes.CallableFromSourceFile):
             return f"{value.source_file}:{value.callable_name}"
         else:
             raise TypeError(f"{value} has unexpected type {type(value)}")
 
 
+class CallableSource(Union):
+    def __init__(self, **kwargs):
+        super().__init__([CallableFromModule(), CallableFromSourceFile()], **kwargs)
+
+
 class Kwargs(Dict):
     def __init__(
         self,

bioimageio/spec/shared/node_transformer.py

@@ -241,14 +241,14 @@ def __init__(self, *, root: typing.Union[os.PathLike, URI]):
         else:
             self.root = pathlib.Path(root).resolve()
 
-    def transform_ImportableSourceFile(
-        self, node: raw_nodes.ImportableSourceFile, **kwargs
-    ) -> raw_nodes.ImportableSourceFile:
+    def transform_CallableFromSourceFile(
+        self, node: raw_nodes.CallableFromSourceFile, **kwargs
+    ) -> raw_nodes.CallableFromSourceFile:
         if isinstance(node.source_file, pathlib.Path) and node.source_file.is_absolute():
             if not isinstance(self.root, pathlib.Path):
                 raise TypeError(f"Cannot convert absolute path '{node.source_file}' with URI root '{self.root}'")
             sf = node.source_file.relative_to(self.root)
-            return raw_nodes.ImportableSourceFile(source_file=sf, callable_name=node.callable_name)
+            return raw_nodes.CallableFromSourceFile(source_file=sf, callable_name=node.callable_name)
         else:
             return node
 
@@ -302,21 +302,21 @@ def transform_PosixPath(self, leaf: pathlib.PosixPath, **kwargs) -> typing.Union
     def transform_WindowsPath(self, leaf: pathlib.WindowsPath, **kwargs) -> typing.Union[URI, pathlib.Path]:
         return self._transform_Path(leaf)
 
-    def transform_ImportableSourceFile(
-        self, node: raw_nodes.ImportableSourceFile, **kwargs
-    ) -> raw_nodes.ImportableSourceFile:
+    def transform_CallableFromSourceFile(
+        self, node: raw_nodes.CallableFromSourceFile, **kwargs
+    ) -> raw_nodes.CallableFromSourceFile:
         if isinstance(node.source_file, URI):
             return node
         elif isinstance(node.source_file, pathlib.Path):
             if node.source_file.is_absolute():
                 return node
             else:
-                return raw_nodes.ImportableSourceFile(
+                return raw_nodes.CallableFromSourceFile(
                     source_file=self.root / node.source_file, callable_name=node.callable_name
                 )
         else:
             raise TypeError(
-                f"Unexpected type '{type(node.source_file)}' for raw_nodes.ImportableSourceFile.source_file '{node.source_file}'"
+                f"Unexpected type '{type(node.source_file)}' for raw_nodes.CallableFromSourceFile.source_file '{node.source_file}'"
             )
 
 
@@ -339,13 +339,15 @@ def transform_URI(
             local_path = _resolve_source(node, root_path=self.root)
             return local_path
 
-    def transform_ImportableSourceFile(
-        self, node: raw_nodes.ImportableSourceFile, **kwargs
-    ) -> raw_nodes.ResolvedImportableSourceFile:
-        return raw_nodes.ResolvedImportableSourceFile(
+    def transform_CallableFromSourceFile(
+        self, node: raw_nodes.CallableFromSourceFile, **kwargs
+    ) -> raw_nodes.ResolvedCallableFromSourceFile:
+        return raw_nodes.ResolvedCallableFromSourceFile(
             source_file=_resolve_source(node.source_file, self.root), callable_name=node.callable_name
         )
 
-    def transform_ImportableModule(self, node: raw_nodes.ImportableModule, **kwargs) -> raw_nodes.LocalImportableModule:
+    def transform_CallableFromModule(
+        self, node: raw_nodes.CallableFromModule, **kwargs
+    ) -> raw_nodes.LocalCallableFromModule:
         r = self.root if isinstance(self.root, pathlib.Path) else pathlib.Path()
-        return raw_nodes.LocalImportableModule(**dataclasses.asdict(node), root_path=r)
+        return raw_nodes.LocalCallableFromModule(**dataclasses.asdict(node), root_path=r)