refactor: Dockerfile installs from pyproject.toml

.github/workflows/ci.yml

@@ -26,6 +26,7 @@ jobs:
       - name: Run make install
         if: steps.cache-venv.outputs.cache-hit != 'true'
         run: make install
+      - run: make test
       - name: install k3d
         run: curl -s https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | bash
       # - uses: mxschmitt/action-tmate@v3

Dockerfile

@@ -1,15 +1,13 @@
 FROM prefecthq/prefect:2.14.9-python3.10
 
-# TODO: unify with requirements in pyproject.toml
-# s3fs is used to fetch flows from s3 remote storage
-# prefect-aws is used to pull flows from s3
-# dask dependences are needed by dask_kubes_flow
-# bokeh is needed for the dask dashboard
-RUN pip install --no-cache-dir bokeh==2.4.3 dask_kubernetes==2023.6.1 prefect-dask==0.2.6 s3fs==2023.6.0 prefect-aws==0.4.5
-
 # explicitly specify workdir expected by the deployment (even though its the same as the base image)
 WORKDIR /opt/prefect
 
+# install python dependencies
+# dask dependences are needed by dask_kubes_flow
+COPY pyproject.toml ./
+RUN pip install --no-cache-dir -e '.[dask]'
+
 # point fsspec (ie: s3fs) at minio inside the cluster
 COPY config/fsspec-minio.json config/fsspec.json
 ENV FSSPEC_CONFIG_DIR=/opt/prefect/config

Makefile-common.mk

@@ -17,7 +17,7 @@ $(pip): $(if $(value CI),|,) .python-version
 	$(pip) install pip~=23.3 wheel~=0.40
 
 $(venv): $(if $(value CI),|,) pyproject.toml $(pip)
-	$(pip) install -e '.[dev]'
+	$(pip) install -e '.[dev,dask]'
 	touch $(venv)
 
 node_modules: package.json

pyproject.toml

@@ -5,17 +5,11 @@ version = "0.0.0"
 readme = "README.md"
 requires-python = ">=3.10"
 dependencies = [
-    # bokeh is needed for the dask dashboard
-    "bokeh==2.4.3",
-    "dask_kubernetes==2023.6.1",
     "prefect==2.14.9",
-    # to push flows to s3
+    # to pull/push flows in s3
     "prefect-aws==0.4.5",
-    "prefect-dask==0.2.6",
-    "prefect-ray==0.3.2",
     "prefect-shell==0.2.2",
-    "ray==2.5.1",
-    # to write flows to s3
+    # to read/write flows in s3 remote storage
     "s3fs~=2023.6.0",
 ]
 
@@ -28,6 +22,13 @@ dev = [
     "ruff~=0.1.6",
     "twine~=4.0",
 ]
+dask = [
+    # bokeh is needed for the dask dashboard
+    "bokeh==2.4.3",
+    "dask_kubernetes==2023.6.1",
+    "prefect-dask==0.2.6",
+]
+ray = ["prefect-ray==0.3.2", "ray==2.5.1"]
 
 [build-system]
 requires = ["setuptools~=68.2", "wheel~=0.40"]