From 83c45e10a5f5c5fa525c318ef96a9b8f58365897 Mon Sep 17 00:00:00 2001
From: Oliver Mannion <125105+tekumara@users.noreply.github.com>
Date: Sat, 2 Dec 2023 15:35:32 +1100
Subject: [PATCH] refactor: Dockerfile installs from pyproject.toml

---
 .github/workflows/ci.yml |  1 +
 Dockerfile               | 12 +++++-------
 Makefile-common.mk       |  2 +-
 pyproject.toml           | 17 +++++++++--------
 4 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 806ad5a..d5a46cd 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -26,6 +26,7 @@ jobs:
       - name: Run make install
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: make install
+      - run: make test
       - name: install k3d
         run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
       # - uses: mxschmitt/action-tmate@v3
diff --git a/Dockerfile b/Dockerfile
index a37f16d..738ee8e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,15 +1,13 @@
 FROM prefecthq/prefect:2.14.9-python3.10
 
-# TODO: unify with requirements in pyproject.toml
-# s3fs is used to fetch flows from s3 remote storage
-# prefect-aws is used to pull flows from s3
-# dask dependences are needed by dask_kubes_flow
-# bokeh is needed for the dask dashboard
-RUN pip install --no-cache-dir bokeh==2.4.3 dask_kubernetes==2023.6.1 prefect-dask==0.2.6 s3fs==2023.6.0 prefect-aws==0.4.5
-
 # explicitly specify workdir expected by the deployment (even though its the same as the base image)
 WORKDIR /opt/prefect
 
+# install python dependencies
+# dask dependences are needed by dask_kubes_flow
+COPY pyproject.toml ./
+RUN pip install --no-cache-dir -e '.[dask]'
+
 # point fsspec (ie: s3fs) at minio inside the cluster
 COPY config/fsspec-minio.json config/fsspec.json
 ENV FSSPEC_CONFIG_DIR=/opt/prefect/config
diff --git a/Makefile-common.mk b/Makefile-common.mk
index 7b82a71..6bb92a7 100644
--- a/Makefile-common.mk
+++ b/Makefile-common.mk
@@ -17,7 +17,7 @@ $(pip): $(if $(value CI),|,) .python-version
 	$(pip) install pip~=23.3 wheel~=0.40
 
 $(venv): $(if $(value CI),|,) pyproject.toml $(pip)
-	$(pip) install -e '.[dev]'
+	$(pip) install -e '.[dev,dask]'
 	touch $(venv)
 
 node_modules: package.json
diff --git a/pyproject.toml b/pyproject.toml
index 8adf058..f24bcd2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -5,17 +5,11 @@ version = "0.0.0"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    # bokeh is needed for the dask dashboard
-    "bokeh==2.4.3",
-    "dask_kubernetes==2023.6.1",
     "prefect==2.14.9",
-    # to push flows to s3
+    # to pull/push flows in s3
     "prefect-aws==0.4.5",
-    "prefect-dask==0.2.6",
-    "prefect-ray==0.3.2",
     "prefect-shell==0.2.2",
-    "ray==2.5.1",
-    # to write flows to s3
+    # to read/write flows in s3 remote storage
     "s3fs~=2023.6.0",
 ]
 
@@ -28,6 +22,13 @@ dev = [
     "ruff~=0.1.6",
     "twine~=4.0",
 ]
+dask = [
+    # bokeh is needed for the dask dashboard
+    "bokeh==2.4.3",
+    "dask_kubernetes==2023.6.1",
+    "prefect-dask==0.2.6",
+]
+ray = ["prefect-ray==0.3.2", "ray==2.5.1"]
 
 [build-system]
 requires = ["setuptools~=68.2", "wheel~=0.40"]