Add support for shared resources.

.github/workflows/test.yml

@@ -47,6 +47,8 @@ jobs:
           hatch run cov-ci
       - name: Lint
         run: |
+          # https://github.com/python/mypy/issues/10863
+          mkdir -p .mypy_cache
           hatch run lint:all
       - name: Test install
         run: |

.gitignore

@@ -6,3 +6,4 @@ example/draft/
 .idea
 /dist
 docs/_build
+.mypy_cache

pyproject.toml

@@ -49,6 +49,7 @@ dependencies = [
   "coverage[toml]>=6.5",
   "pytest",
   "pytest_mock",
+  "pytest-unordered",
   "sphinx",
   "sphinx-rtd-theme",
   "myst-parser",
@@ -202,6 +203,9 @@ exclude_lines = [
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 
+[tool.mypy]
+allow_redefinition = true
+
 # https://github.com/libgit2/pygit2/issues/709
 [[tool.mypy.overrides]]
 module = "pygit2"

src/orderly/__init__.py

@@ -1,3 +1,17 @@
-from orderly.core import artefact, dependency, description, parameters, resource
+from orderly.core import (
+    artefact,
+    dependency,
+    description,
+    parameters,
+    resource,
+    shared_resource,
+)
 
-__all__ = ["artefact", "dependency", "description", "parameters", "resource"]
+__all__ = [
+    "artefact",
+    "dependency",
+    "description",
+    "parameters",
+    "resource",
+    "shared_resource",
+]

src/orderly/core.py

@@ -1,4 +1,7 @@
+import os.path
+import shutil
 from dataclasses import dataclass
+from pathlib import Path
 from typing import Dict, List, Union
 
 from dataclasses_json import dataclass_json
@@ -73,6 +76,74 @@ def resource(files):
     return files_expanded
 
 
+def shared_resource(
+    files: Union[str, List[str], Dict[str, str]]
+) -> Dict[str, str]:
+    """Copy shared resources into a packet directory.
+
+    You can use this to share common resources (data or code) between multiple
+    packets. Additional metadata will be added to keep track of where the files
+    came from. Using this function requires the shared resources directory
+    `shared/` exists at the orderly root; an error will be
+    raised if this is not configured when we attempt to fetch files.
+
+    Parameters
+    ----------
+    files: str | [str] | dict[str, str]
+        The shared resources to copy. If a dictionary is provided, the keys will
+        be the destination file while the value is the filename within the
+        shared resource directory.
+
+    Returns
+    -------
+        A dictionary of files that were copied. If a directory was copied, this
+        includes all of its individual contents. Do not rely on the ordering
+        where directory expansion was performed, as it may be platform
+        dependent.
+    """
+    ctx = get_active_context()
+
+    files = util.relative_path_mapping(files, "shared resource")
+    result = _copy_shared_resources(ctx.root.path, ctx.path, files)
+
+    if ctx.is_active:
+        for f in result.keys():
+            ctx.packet.mark_file_immutable(f)
+        ctx.orderly.shared_resources.update(result)
+
+    return result
+
+
+def _copy_shared_resources(
+    root: Path, packet: Path, files: Dict[str, str]
+) -> Dict[str, str]:
+    shared_path = root / "shared"
+    if not shared_path.exists():
+        msg = "The shared resources directory 'shared' does not exist at orderly's root"
+        raise Exception(msg)
+
+    result = {}
+    for here, there in files.items():
+        src = shared_path / there
+        dst = packet / here
+
+        if not src.exists():
+            msg = f"Shared resource file '{there}' does not exist. Looked within directory '{shared_path}'"
+            raise Exception(msg)
+        elif src.is_dir():
+            shutil.copytree(src, dst, dirs_exist_ok=True)
+            copied = {
+                os.path.join(here, f): os.path.join(there, f)
+                for f in util.all_normal_files(src)
+            }
+            result.update(copied)
+        else:
+            shutil.copyfile(src, dst)
+            result[here] = there
+
+    return result
+
+
 def artefact(name, files):
     """Declare an artefact.
 

src/orderly/current.py

@@ -10,6 +10,7 @@
 class OrderlyCustomMetadata:
     def __init__(self):
         self.resources = []
+        self.shared_resources = {}
         self.artefacts = []
         self.description = None
 

src/orderly/run.py

@@ -123,11 +123,14 @@ def _custom_metadata(entrypoint, orderly):
     role = [{"path": entrypoint, "role": "orderly"}]
     for p in orderly.resources:
         role.append({"path": p, "role": "resource"})
+    for p in orderly.shared_resources:
+        role.append({"path": p, "role": "shared"})
 
     return {
         "role": role,
         "artefacts": orderly.artefacts,
         "description": orderly.description or Description.empty(),
+        "shared": orderly.shared_resources,
     }
 
 

src/outpack/util.py

@@ -6,7 +6,7 @@
 from contextlib import contextmanager
 from itertools import filterfalse, tee
 from pathlib import Path
-from typing import Optional
+from typing import Dict, List, Optional, Union
 
 
 def find_file_descend(filename, path):
@@ -82,6 +82,28 @@ def assert_file_exists(path, *, workdir=None, name="File"):
         raise Exception(msg)
 
 
+def assert_relative_path(path: str, name: str):
+    path = Path(path)
+
+    # On Windows, this is not equivalent to path.is_absolute().
+    # There are paths which are neither absolute nor relative, but somewhere
+    # between the two. These are paths such as `C:foo` or `\foo`. The former has
+    # a drive but no root, and the latter has a root but no drive. We want to
+    # exclude both scenarios. On POSIX, drive will always be empty and this is
+    # equivalent to calling `is_absolute()`.
+    #
+    # See https://github.com/python/cpython/issues/44626 for some discussion.
+    # Unfortunately, while the issue was closed, the `is_relative` function
+    # mentioned was never added.
+    if path.drive or path.root:
+        msg = f"Expected {name} path '{path}' to be a relative path"
+        raise Exception(msg)
+
+    if ".." in path.parts:
+        msg = f"Path '{path}' must not contain '..' component"
+        raise Exception(msg)
+
+
 def expand_dirs(paths, *, workdir=None):
     if len(paths) == 0:
         return []
@@ -102,13 +124,28 @@ def match_value(arg, choices, name):
         raise Exception(msg)
 
 
-def relative_path_array(files, name):
+def relative_path_array(files: Union[str, List[str]], name: str) -> List[str]:
     if not isinstance(files, list):
         files = [files]
+
     for f in files:
-        if os.path.isabs(f):
-            msg = f"Expected {name} path '{f}' to be a relative path"
-            raise Exception(msg)
+        assert_relative_path(f, name)
+
+    return files
+
+
+def relative_path_mapping(
+    files: Union[str, List[str], Dict[str, str]], name: str
+) -> Dict[str, str]:
+    if isinstance(files, str):
+        files = {files: files}
+    elif isinstance(files, list):
+        files = {f: f for f in files}
+
+    for k, v in files.items():
+        assert_relative_path(k, name)
+        assert_relative_path(v, name)
+
     return files
 
 

tests/helpers/__init__.py

@@ -93,6 +93,21 @@ def copy_examples(names, root):
         shutil.copytree(Path("tests/orderly/examples") / nm, path_src)
 
 
+def copy_shared_resources(names, root):
+    shared_path = root.path / "shared"
+    shared_path.mkdir(exist_ok=True)
+
+    if isinstance(names, str):
+        names = [names]
+    for nm in names:
+        src = Path("tests/orderly/shared/") / nm
+        dst = shared_path / nm
+        if src.is_dir():
+            shutil.copytree(src, dst)
+        else:
+            shutil.copyfile(src, dst)
+
+
 def create_metadata_depends(id: str, depends: Optional[List[str]] = None):
     if depends is None:
         depends = []

tests/orderly/examples/shared/shared.py

@@ -0,0 +1,9 @@
+import orderly
+
+orderly.shared_resource({"shared_data.txt": "numbers.txt"})
+
+with open("shared_data.txt") as f:
+    dat = [int(x) for x in f.readlines()]
+
+with open("result.txt", "w") as f:
+    f.write(f"{sum(dat)}\n")

tests/orderly/examples/shared_dir/shared_dir.py

@@ -0,0 +1,14 @@
+from operator import mul
+
+import orderly
+
+orderly.shared_resource({"shared_data": "data"})
+
+with open("shared_data/numbers.txt") as f:
+    numbers = [float(x) for x in f.readlines()]
+with open("shared_data/weights.txt") as f:
+    weights = [float(x) for x in f.readlines()]
+
+result = sum(map(mul, numbers, weights))
+with open("result.txt", "w") as f:
+    f.write(f"{result}\n")

tests/orderly/shared/data/numbers.txt

@@ -0,0 +1,5 @@
+1
+2
+3
+4
+5

tests/orderly/shared/data/weights.txt

@@ -0,0 +1,5 @@
+2.8
+9.9
+1.1
+0.5
+4.2

tests/orderly/shared/numbers.txt

@@ -0,0 +1,5 @@
+1
+2
+3
+4
+5