Merge pull request mmistakes#253 from jhaux/betta_meta

Betta meta

CHANGELOG.md

@@ -6,6 +6,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 ### Added
+- Root parameter for image and numpy loader of the meta dataset. `root` is prepended to the given paths and thus allows for smaller label arrays
+- Category loader allows to convert a given label into a more expressive category, which is specifed in the dataset's `meta.yaml`
 - Debug options: `debug/disable_integrations=True`, `debug/max_examples=5 batches`.
 - Epoch and Batch step are restored.
 - Added option to save checkpoint zero with `--ckpt_zero True`.

edflow/data/believers/meta.py

@@ -6,7 +6,7 @@
 from edflow.data.dataset_mixin import DatasetMixin
 from edflow.util import retrieve, get_obj_from_str, pp2mkdtable, pop_keypath
 from edflow.util import walk, set_value, edprint
-from edflow.data.believers.meta_loaders import image_loader, numpy_loader
+from edflow.data.believers.meta_loaders import DEFAULT_LOADERS
 
 try:
     from IPython import get_ipython
@@ -17,9 +17,6 @@
     __COULD_HAVE_IPYTHON__ = False
 
 
-DEFAULT_LOADERS = {"image": image_loader, "np": numpy_loader}
-
-
 class MetaDataset(DatasetMixin):
     """
     The :class:`MetaDataset` allows for easy data reading using a simple

edflow/data/believers/meta_loaders.py

@@ -1,15 +1,20 @@
 import numpy as np
+import os
 from PIL import Image
 from edflow.data.util import adjust_support
 
 
-def image_loader(path, support="0->255", resize_to=None):
+def image_loader(path, root="", support="0->255", resize_to=None):
     """
 
     Parameters
     ----------
     path : str
         Where to finde the image.
+    root : str
+        Root path, at which the suuplied :attr:`path` starts. E.g. if all paths
+        supplied to this function are relative to
+        ``/export/scratch/you_are_great/dataset``, this path would be root.
     support : str
         Defines the support and data type of the loaded image. Must be one of
             - ``0->255``: The PIL default. Datatype is ``np.uint8`` and all values
@@ -30,8 +35,10 @@ def image_loader(path, support="0->255", resize_to=None):
         specified.
     """
 
-    def loader(support=support, resize_to=resize_to):
-        im = Image.open(path)
+    def loader(support=support, resize_to=resize_to, root=root):
+        path_ = os.path.join(root, path)
+
+        im = Image.open(path_)
 
         if resize_to is not None:
             if isinstance(resize_to, int):
@@ -46,10 +53,14 @@ def loader(support=support, resize_to=resize_to):
         else:
             return adjust_support(im, support, "0->255")
 
+    loader.__doc__ = f"""Loads the image found at {path} relative to {root},
+    scales the support to :attr:`support` (default={support}) and resizes the
+    image to :attr:`resize_to` (default: {resize_to}."""
+
     return loader
 
 
-def numpy_loader(path):
+def numpy_loader(path, root=""):
     """
 
     Parameters
@@ -64,6 +75,66 @@ def numpy_loader(path):
     """
 
     def loader():
-        return np.load(path)
+        path_ = os.path.join(root, path)
+        return np.load(path_)
+
+    loader.__doc__ = f"""Loads the numpy array found at {path} relative to
+    {root}."""
 
     return loader
+
+
+def category(index, categories):
+    """
+    Turns an abstract category label into a readable label.
+
+    Example:
+
+    Your dataset has the label ``pid`` which has integer entries like ``[0, 0,
+    0, ..., 2, 2]`` between 0 and 3.
+
+    Inside the dataset's ``meta.yaml`` you define
+
+    .. code-block:: yaml
+
+        # meta.yaml
+        # ...
+        loaders:
+            pid: category
+        loader_kwargs:
+            pid:
+                categories: ['besser', 'pesser', 'Mimo Tilbich']
+
+    Now examples will be annotated with ``{pid: 'besser'}`` if the pid is
+    ``0``, ``{pid: 'pesser'}`` if pid is 1 or ``{pid: 'Mimo Tilbich'}`` if the
+    pid is 2.
+
+    Note that categories can be anything that implements a ``__getitem__``
+    method. You simply need to take care, that it understands the ``index``
+    value it is passed by this loader function.
+
+    Parameters
+    ----------
+    index : int, Hashable
+        Some value that will be passed to :attr:`categories`'s
+        :meth:`__getitem__` method. I.e. :attr:`categories` can be a ``list`` or
+        ``dict`` or whatever you want!
+    categories : list, dict, object with `__getitem__` method
+        Defines the categories you have in you dataset. Will be accessed like
+        ``categories[index]``
+
+    Returns
+    -------
+        category : object
+            ``categories[index]``
+    """
+
+    return categories[index]
+
+
+DEFAULT_LOADERS = {
+    "image": image_loader,
+    "np": numpy_loader,
+    "category": category,
+    "cat": category,
+}