Added MacOS compatibility

.github/workflows/ci-tests.yaml

@@ -24,9 +24,9 @@ jobs:
       - uses: github/codeql-action/analyze@v2
   cwl-conformance:
     name: "CWL conformance tests"
-    runs-on: ubuntu-22.04
     strategy:
       matrix:
+        on: [ "ubuntu-22.04" ]
         python: [ "3.8", "3.9", "3.10", "3.11" ]
         version: [ "v1.0", "v1.1", "v1.2" ]
         include:
@@ -39,6 +39,10 @@ jobs:
           - commit: "ad6f77c648ae9f0eaa2fd53ba7032b0523e12de8"
             exclude: "docker_entrypoint,modify_file_content"
             version: "v1.2"
+          - on: "macos-12"
+            python: "3.11"
+            version: "v1.2"
+    runs-on: ${{ matrix.on }}
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -47,6 +51,8 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: "15"
+      - uses: douglascamata/setup-docker-macos-action@v1-alpha
+        if: ${{ startsWith(matrix.on, 'macos-') }}
       - uses: docker/setup-qemu-action@v2
       - name: "Install Streamflow"
         run: |
@@ -89,10 +95,14 @@ jobs:
         run: tox
   unit-tests:
     name: "StreamFlow unit tests"
-    runs-on: ubuntu-22.04
     strategy:
       matrix:
+        on: [ "ubuntu-22.04"]
         python: [ "3.8", "3.9", "3.10", "3.11" ]
+        include:
+          - on: "macos-12"
+            python: "3.11"
+    runs-on: ${{ matrix.on }}
     env:
       TOXENV: ${{ format('py{0}-unit', matrix.python) }}
     steps:
@@ -107,13 +117,15 @@ jobs:
       - uses: actions/setup-node@v3
         with:
           node-version: "15"
+      - uses: douglascamata/setup-docker-macos-action@v1-alpha
+        if: ${{ startsWith(matrix.on, 'macos-') }}
       - uses: docker/setup-qemu-action@v2
       - name: "Install Python Dependencies and Streamflow"
         run: |
           python -m pip install tox --user
           python -m pip install . --user
       - name: "Run StreamFlow tests via Tox"
-        run: tox
+        run: python -m tox
       - name: "Upload coverage report for unit tests"
         uses: actions/upload-artifact@v3
         with:
@@ -133,4 +145,4 @@ jobs:
         uses: codecov/codecov-action@v3
         with:
           fail_ci_if_error: true
-          token: ${{ secrets.CODECOV_TOKEN }}
+          token: ${{ secrets.CODECOV_TOKEN }}

streamflow/core/deployment.py

@@ -27,7 +27,7 @@ def _init_workdir(deployment_name: str) -> str:
     if deployment_name != LOCAL_LOCATION:
         return posixpath.join("/tmp", "streamflow")  # nosec
     else:
-        return os.path.join(tempfile.gettempdir(), "streamflow")
+        return os.path.join(os.path.realpath(tempfile.gettempdir()), "streamflow")
 
 
 class Location:

streamflow/core/utils.py

@@ -97,9 +97,10 @@ def dict_product(**kwargs) -> MutableMapping[Any, Any]:
         yield dict(zip(keys, list(instance)))
 
 
-def encode_command(command: str):
-    return "echo {command} | base64 -d | sh".format(
-        command=base64.b64encode(command.encode("utf-8")).decode("utf-8")
+def encode_command(command: str, shell: str = "sh"):
+    return "echo {command} | base64 -d | {shell}".format(
+        command=base64.b64encode(command.encode("utf-8")).decode("utf-8"),
+        shell=shell,
     )
 
 

streamflow/cwl/translator.py

@@ -1045,7 +1045,9 @@ def _process_docker_requirement(
     if "dockerOutputDirectory" in docker_requirement:
         docker_config["workdir"] = docker_requirement["dockerOutputDirectory"]
         context["output_directory"] = docker_config["workdir"]
-        local_dir = os.path.join(tempfile.gettempdir(), "streamflow", random_name())
+        local_dir = os.path.join(
+            os.path.realpath(tempfile.gettempdir()), "streamflow", random_name()
+        )
         os.makedirs(local_dir, exist_ok=True)
         docker_config["volume"].append(f"{local_dir}:{docker_config['workdir']}")
     # Build step target

streamflow/deployment/connector/base.py

@@ -249,6 +249,9 @@ def _get_run_command(
     ) -> str:
         ...
 
+    def _get_shell(self) -> str:
+        return "sh"
+
     def _get_stream_reader(self, location: Location, src: str) -> StreamWrapperContext:
         dirname, basename = posixpath.split(src)
         return SubprocessStreamReaderWrapperContext(
@@ -369,7 +372,7 @@ async def run(
                     job=f"for job {job_name}" if job_name else "",
                 )
             )
-        command = utils.encode_command(command)
+        command = utils.encode_command(command, self._get_shell())
         run_command = self._get_run_command(command, location)
         proc = await asyncio.create_subprocess_exec(
             *shlex.split(run_command),

streamflow/deployment/connector/local.py

@@ -33,9 +33,17 @@ def _get_run_command(
         self, command: str, location: Location, interactive: bool = False
     ):
         if sys.platform == "win32":
-            return f"cmd /C '{command}'"
+            return f"{self._get_shell()} /C '{command}'"
         else:
-            return f"sh -c '{command}'"
+            return f"{self._get_shell()} -c '{command}'"
+
+    def _get_shell(self) -> str:
+        if sys.platform == "win32":
+            return "cmd"
+        elif sys.platform == "darwin":
+            return "bash"
+        else:
+            return "sh"
 
     async def _copy_remote_to_remote(
         self,
@@ -64,7 +72,10 @@ async def _copy_remote_to_remote(
             )
 
     async def deploy(self, external: bool) -> None:
-        os.makedirs(os.path.join(tempfile.gettempdir(), "streamflow"), exist_ok=True)
+        os.makedirs(
+            os.path.join(os.path.realpath(tempfile.gettempdir()), "streamflow"),
+            exist_ok=True,
+        )
 
     async def get_available_locations(
         self,

streamflow/recovery/checkpoint_manager.py

@@ -22,7 +22,10 @@ class DefaultCheckpointManager(CheckpointManager):
     def __init__(self, context: StreamFlowContext, checkpoint_dir: str | None = None):
         super().__init__(context)
         self.checkpoint_dir = checkpoint_dir or os.path.join(
-            tempfile.gettempdir(), "streamflow", "checkpoint", utils.random_name()
+            os.path.realpath(tempfile.gettempdir()),
+            "streamflow",
+            "checkpoint",
+            utils.random_name(),
         )
         self.copy_tasks: MutableSequence = []
 

tests/conftest.py

@@ -1,4 +1,5 @@
 import asyncio
+import os
 import tempfile
 from asyncio.locks import Lock
 from collections.abc import Iterable
@@ -42,7 +43,7 @@ def get_docker_deployment_config():
 @pytest_asyncio.fixture(scope="session")
 async def context() -> StreamFlowContext:
     context = build_context(
-        tempfile.gettempdir(),
+        os.path.realpath(tempfile.gettempdir()),
         {"database": {"type": "default", "config": {"connection": ":memory:"}}},
     )
     await context.deployment_manager.deploy(
@@ -52,7 +53,7 @@ async def context() -> StreamFlowContext:
             config={},
             external=True,
             lazy=False,
-            workdir=tempfile.gettempdir(),
+            workdir=os.path.realpath(tempfile.gettempdir()),
         )
     )
     await context.deployment_manager.deploy(get_docker_deployment_config())