Provide frozen requirements.txt files for images, and automation to update them

.github/workflows/refreeze-dockerfile-requirements-txt.yaml

@@ -0,0 +1,61 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps.
+# ref: https://docs.github.com/en/actions/learn-github-actions/workflow-syntax-for-github-actions
+#
+name: Refreeze Dockerfile.requirements.txt
+
+on:
+  push:
+    paths:
+      - "**/Dockerfile"
+      - "**/Dockerfile.requirements.in"
+      - "**/Dockerfile.requirements.txt"
+      - ".github/workflows/refreeze-dockerfile-requirements-txt.yaml"
+    branches: ["main"]
+  workflow_dispatch:
+
+jobs:
+  refreeze-dockerfile-requirements-txt:
+    name: Refreeze Dockerfile.requirements.txt
+
+    # Don't run this job on forks
+    if: github.repository == 'dask/dask-gateway'
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - image: dask-gateway
+          - image: dask-gateway-server
+
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Refreeze Dockerfile.requirements.txt based on Dockerfile.requirements.in
+        run: |
+          cd ${{ matrix.image }}
+          docker run --rm \
+              --env=CUSTOM_COMPILE_COMMAND='Use "Run workflow" button at https://github.com/dask/dask-gateway/actions/workflows/refreeze-dockerfile-requirements-txt.yaml' \
+              --volume=$PWD:/opt/${{ matrix.image }} \
+              --workdir=/opt/${{ matrix.image }} \
+              --user=root \
+              python:3.10-slim-bullseye \
+              sh -c 'pip install pip-tools==6.* && pip-compile --upgrade --output-file=Dockerfile.requirements.txt Dockerfile.requirements.in'
+
+      - name: git diff
+        run: git --no-pager diff --color=always
+
+      # ref: https://github.com/peter-evans/create-pull-request
+      - name: Create a PR
+        uses: peter-evans/create-pull-request@v4
+        with:
+          token: "${{ secrets.dask_bot_token }}"
+          author: Dask Bot Account <65357765+dask-bot@users.noreply.github.com>
+          committer: Dask Bot Account <65357765+dask-bot@users.noreply.github.com>
+          branch: update-image-${{ matrix.image }}
+          labels: dependencies
+          commit-message: Refreeze ${{ matrix.image}}/Dockerfile.requirements.txt
+          title: Refreeze ${{ matrix.image}}/Dockerfile.requirements.txt
+          body: >-
+            ${{ matrix.image}}/Dockerfile.requirements.txt has been refrozen
+            based on ${{ matrix.image}}/Dockerfile.requirements.in.

RELEASE.md

@@ -2,6 +2,11 @@
 
 Follow these steps to cut a new release.
 
+- Update frozen Dockerfile.requirements.txt files by running workflow that has
+  automated this at
+  https://github.com/dask/dask-gateway/actions/workflows/refreeze-dockerfile-requirements-txt.yaml
+  and merging the PRs.
+
 - Update `docs/changelog.md` in a dedicated PR
 
   - Generate a list of PRs using [executablebooks/github-activity](https://github.com/executablebooks/github-activity)

dask-gateway-server/Dockerfile

@@ -33,10 +33,9 @@ WORKDIR /home/dask/
 # The Golang proxy binary isn't built as the dask-gateway Helm chart relies on
 # Traefik as a proxy instead to run in its dedicated pod.
 #
-COPY --chown=dask:dask . /srv/dask-gateway-server
-
+COPY --chown=dask:dask . /opt/dask-gateway-server
 RUN DASK_GATEWAY_SERVER__NO_PROXY=true pip install --no-cache-dir \
-        "/srv/dask-gateway-server[kubernetes]"
+        -r /opt/dask-gateway-server/Dockerfile.requirements.txt
 
 ENTRYPOINT ["tini", "-g", "--"]
 CMD ["dask-gateway-server", "--config", "/etc/dask-gateway/dask_gateway_config.py"]

dask-gateway-server/Dockerfile.requirements.in

@@ -0,0 +1,9 @@
+# These are the requirements we know we want to install in the Dockerfile, and
+# then we freeze them ahead of time to provide a clear description of the
+# dependencies we have installed.
+#
+
+# Install dask-gateway-server[kubernetes], which is the only thing needed for
+# our CI test suite.
+#
+.[kubernetes]

dask-gateway-server/Dockerfile.requirements.txt

@@ -0,0 +1,59 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    Use "Run workflow" button at https://github.com/dask/dask-gateway/actions/workflows/refreeze-dockerfile-requirements-txt.yaml
+#
+aiohttp==3.8.1
+    # via
+    #   dask-gateway-server
+    #   kubernetes-asyncio
+aiosignal==1.2.0
+    # via aiohttp
+async-timeout==4.0.2
+    # via aiohttp
+attrs==21.4.0
+    # via aiohttp
+certifi==2022.5.18.1
+    # via kubernetes-asyncio
+cffi==1.15.0
+    # via cryptography
+charset-normalizer==2.0.12
+    # via aiohttp
+colorlog==6.6.0
+    # via dask-gateway-server
+cryptography==37.0.2
+    # via dask-gateway-server
+dask-gateway-server @ file:///opt/dask-gateway-server
+    # via -r Dockerfile.requirements.in
+frozenlist==1.3.0
+    # via
+    #   aiohttp
+    #   aiosignal
+idna==3.3
+    # via yarl
+kubernetes-asyncio==22.6.5
+    # via dask-gateway-server
+multidict==6.0.2
+    # via
+    #   aiohttp
+    #   yarl
+pycparser==2.21
+    # via cffi
+python-dateutil==2.8.2
+    # via kubernetes-asyncio
+pyyaml==6.0
+    # via kubernetes-asyncio
+six==1.16.0
+    # via
+    #   kubernetes-asyncio
+    #   python-dateutil
+traitlets==5.2.2.post1
+    # via dask-gateway-server
+urllib3==1.26.9
+    # via kubernetes-asyncio
+yarl==1.7.2
+    # via aiohttp
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

dask-gateway/Dockerfile

@@ -34,22 +34,8 @@ WORKDIR /home/dask/
 
 # Install dask-gateway
 COPY --chown=dask:dask . /opt/dask-gateway
-# Install dask-gateway, which is the only thing needed for our CI test suite.
-#
-# We also install the bare minimum to provide end users with a primitive
-# end-to-end demonstrative test doing work in the worker pods and accessing the
-# scheduler dashboard without changing the image.
-#
-# - bokeh is required by the scheduler pod to present dashbaords.
-# - numpy is required for running a basic computation test:
-#   https://gateway.dask.org/usage.html#run-computations-on-the-cluster
-#
-COPY --chown=dask:dask . /opt/dask-gateway
-RUN pip install --no-cache-dir
-        /opt/dask-gateway \
- && pip install --no-cache-dir \
-        bokeh \
-        numpy
+RUN pip install --no-cache-dir \
+        -r /opt/dask-gateway/Dockerfile.requirements.txt
 
 # Only set ENTRYPOINT, CMD is configured at runtime by dask-gateway-server
 ENTRYPOINT ["tini", "-g", "--"]

dask-gateway/Dockerfile.requirements.in

@@ -0,0 +1,18 @@
+# These are the requirements we know we want to install in the Dockerfile, and
+# then we freeze them ahead of time to provide a clear description of the
+# dependencies we have installed.
+#
+
+# Install dask-gateway, which is the only thing needed for our CI test suite.
+.
+
+# We also install the bare minimum to provide end users with a primitive
+# end-to-end demonstrative test doing work in the worker pods and accessing the
+# scheduler dashboard without changing the image.
+#
+# - bokeh is required by the scheduler pod to present dashbaords.
+# - numpy is required for running a basic computation test:
+#   https://gateway.dask.org/usage.html#run-computations-on-the-cluster
+#
+bokeh
+numpy

dask-gateway/Dockerfile.requirements.txt

@@ -0,0 +1,105 @@
+#
+# This file is autogenerated by pip-compile with python 3.10
+# To update, run:
+#
+#    Use "Run workflow" button at https://github.com/dask/dask-gateway/actions/workflows/refreeze-dockerfile-requirements-txt.yaml
+#
+aiohttp==3.8.1
+    # via dask-gateway
+aiosignal==1.2.0
+    # via aiohttp
+async-timeout==4.0.2
+    # via aiohttp
+attrs==21.4.0
+    # via aiohttp
+bokeh==2.4.3
+    # via -r Dockerfile.requirements.in
+charset-normalizer==2.0.12
+    # via aiohttp
+click==8.1.3
+    # via
+    #   dask-gateway
+    #   distributed
+cloudpickle==2.1.0
+    # via
+    #   dask
+    #   distributed
+dask==2022.6.0
+    # via
+    #   dask-gateway
+    #   distributed
+dask-gateway @ file:///opt/dask-gateway
+    # via -r Dockerfile.requirements.in
+distributed==2022.6.0
+    # via dask-gateway
+frozenlist==1.3.0
+    # via
+    #   aiohttp
+    #   aiosignal
+fsspec==2022.5.0
+    # via dask
+heapdict==1.0.1
+    # via zict
+idna==3.3
+    # via yarl
+jinja2==3.1.2
+    # via
+    #   bokeh
+    #   distributed
+locket==1.0.0
+    # via
+    #   distributed
+    #   partd
+markupsafe==2.1.1
+    # via jinja2
+msgpack==1.0.4
+    # via distributed
+multidict==6.0.2
+    # via
+    #   aiohttp
+    #   yarl
+numpy==1.22.4
+    # via
+    #   -r Dockerfile.requirements.in
+    #   bokeh
+packaging==21.3
+    # via
+    #   bokeh
+    #   dask
+    #   distributed
+partd==1.2.0
+    # via dask
+pillow==9.1.1
+    # via bokeh
+psutil==5.9.1
+    # via distributed
+pyparsing==3.0.9
+    # via packaging
+pyyaml==6.0
+    # via
+    #   bokeh
+    #   dask
+    #   dask-gateway
+    #   distributed
+sortedcontainers==2.4.0
+    # via distributed
+tblib==1.7.0
+    # via distributed
+toolz==0.11.2
+    # via
+    #   dask
+    #   distributed
+    #   partd
+tornado==6.1
+    # via
+    #   bokeh
+    #   dask-gateway
+    #   distributed
+typing-extensions==4.2.0
+    # via bokeh
+urllib3==1.26.9
+    # via distributed
+yarl==1.7.2
+    # via aiohttp
+zict==2.2.0
+    # via distributed