Merge pull request #1562 from minrk/freeze-hub

freeze environment in hub image

dev-requirements.txt

@@ -24,3 +24,6 @@ yamllint>=1.17.0
 ## ./dev use these
 colorama
 python-dotenv
+
+# click is used to make cli scripts
+click

images/hub/Dockerfile

@@ -40,7 +40,8 @@ RUN adduser --disabled-password \
     --force-badname \
     ${NB_USER}
 
-ADD requirements.txt /tmp/requirements.txt
+RUN python3 -m pip install --upgrade --no-cache setuptools pip
+COPY requirements.txt /tmp/requirements.txt
 RUN PYCURL_SSL_LIBRARY=openssl pip3 install --no-cache-dir \
          -r /tmp/requirements.txt \
          $(bash -c 'if [[ $JUPYTERHUB_VERSION == "git"* ]]; then \
@@ -57,5 +58,12 @@ RUN chown ${NB_USER}:${NB_USER} /srv/jupyterhub
 # JupyterHub API port
 EXPOSE 8081
 
+# when building the dependencies image
+# add pip-tools necessary for computing dependencies
+# this is not done in production builds by chartpress
+ARG PIP_TOOLS=
+RUN test -z "$PIP_TOOLS" || pip install --no-cache pip-tools==$PIP_TOOLS
+
 USER ${NB_USER}
 CMD ["jupyterhub", "--config", "/etc/jupyterhub/jupyterhub_config.py"]
+

images/hub/dependencies

@@ -0,0 +1,201 @@
+#!/usr/bin/env python3
+"""automatically manage requirements.txt dependencies with pip-tools
+
+See
+
+    ./dependencies --help for commands and arguments
+
+How it works:
+
+- the image used in the helm chart installs a frozen environment from requirements.txt
+- `pip-compile` is used to generate frozen `requirements.txt` from our actual requirements
+  in requirements.in.
+- `pip list --outdated` is used to report available updates for packages
+  in the frozen environment
+- pip-compile etc. are run *inside the image* to ensure consistent behavior,
+  rather than running on host systems, which can vary.
+- When building the image to be used for running dependency-management commands,
+  chartpress configuration is loaded to ensure the environment is the same
+  as when chartpress builds the tagged image to be published.
+"""
+
+from functools import lru_cache
+import json
+import os
+from subprocess import check_call, check_output
+
+import click
+from ruamel.yaml import YAML
+
+yaml = YAML()
+here = os.path.dirname(os.path.abspath(__file__))
+chartpress_yaml = os.path.join(here, os.pardir, os.pardir, "chartpress.yaml")
+values_yaml = os.path.join(here, os.pardir, os.pardir, "jupyterhub", "values.yaml")
+dependencies_image = 'hub-dependencies'
+pip_tools_version="4.*"
+
+
+@lru_cache()
+def build_args(image_name='hub'):
+    """retrieve docker build arguments from chartpress.yaml config file
+
+    Args:
+
+    image_name (str):
+        the name of the image to be built in chartpress.yaml
+    """
+    with open(chartpress_yaml) as f:
+        chartpress_config = yaml.load(f)
+    chart = chartpress_config['charts'][0]
+    image_config = chart['images'][image_name]
+    return image_config.get('buildArgs', {})
+
+
+def build_image():
+    """Build the docker image used for computing dependencies
+
+    This runs the chartpress build of the current image
+    with the addition of pip-tools, used for computing dependencies.
+
+    The image is built with the current frozen environment in requirements.txt
+    and pip-tools commands are available for updating requirements.txt from requirements.in.
+    """
+    click.echo(f"Building docker image {dependencies_image}")
+    build_arg_dict = build_args()
+    build_arg_list = ["--build-arg", f"PIP_TOOLS={pip_tools_version}"]
+    for key in sorted(build_arg_dict):
+        value = build_arg_dict[key]
+        build_arg_list.append("--build-arg")
+        build_arg_list.append(f"{key}={value}")
+    check_call(["docker", "build", "-t", dependencies_image] + build_arg_list + [here])
+
+
+@click.group()
+def cli():
+    """Manage the Python dependencies in this image."""
+    pass
+
+
+@click.command()
+@click.option(
+    '--build/--no-build',
+    help="add --no-build to skip building the dependencies image prior to upgrading",
+    default=True,
+)
+@click.option(
+    '--upgrade/--no-upgrade',
+    help="--upgrade to upgrade all dependencies within the range specified in requirements.in",
+    default=False,
+)
+@click.option(
+    '--upgrade-package',
+    help="specify individual packages to upgrade within the range specified in requirements.in",
+    multiple=True,
+)
+def freeze(build, upgrade, upgrade_package):
+    """Freeze the environment, updating requirements.txt from requirements.in
+
+    Individual packages can be updated, or the whole environment.
+
+    This command:
+
+    1. builds the image with the current frozen environment
+    2. runs pip-compile in the image to update requirements.txt from requirements.in,
+       passing through additional arguments to pip-compile
+    """
+    if build:
+        build_image()
+    click.echo("freezing dependencies with pip-compile")
+    upgrade_args = []
+    if upgrade:
+        upgrade_args.append("--upgrade")
+    for pkg in upgrade_package:
+        upgrade_args.append("--upgrade-package")
+        upgrade_args.append(pkg)
+    check_call(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-it",
+            "-e",
+            "CUSTOM_COMPILE_COMMAND=./dependencies freeze",
+            "--volume",
+            f"{here}:/io",
+            "--workdir",
+            "/io",
+            dependencies_image,
+            "pip-compile",
+        ]
+        + upgrade_args
+    )
+
+
+cli.add_command(freeze)
+
+
+@click.command()
+@click.option('--build/--no-build', default=True)
+def outdated(build):
+    """Check for outdated dependencies with pip.
+
+    This command:
+
+    1. builds the image with the current frozen environment
+    2. runs `pip list --outdated` to report any outdated packages
+       that could be candidates for upgrade
+    """
+    if build:
+        build_image()
+    click.echo("Checking for outdated dependencies with pip.")
+    outdated_json = check_output(
+        [
+            "docker",
+            "run",
+            "--rm",
+            "-it",
+            dependencies_image,
+            "pip",
+            "list",
+            "--outdated",
+            "--format=json",
+        ]
+    ).decode("utf8")
+    outdated = json.loads(outdated_json)
+    have_outdated = False
+    for pkg in outdated:
+        name = pkg['name']
+        # ignore some common packages that aren't relevant to our requirements.txt
+        if name in {'pip', 'setuptools', 'wheel'}:
+            continue
+        have_outdated = True
+        version = pkg['version']
+        latest = pkg['latest_version']
+        # TODO: parser requirements.in to check if latest is in-range?
+        # If they are in-range, running freeze again is enough,
+        # but if they are outside the range, requirements.in needs to be updated
+        # first to pick them up
+        # for now, print as much so humans can decide
+        print(f"Have {name}=={version}, latest is {name}=={latest}")
+
+    if have_outdated:
+        print("There are outdated dependencies!")
+        print(
+            "To pick up any versions outside the range(s) specified in requirements.in,"
+        )
+        print("update the pinning(s) in that file.")
+        print(
+            "To update the whole environment within the given ranges, run `./dependencies freeze --upgrade`"
+        )
+        print(
+            "To update one or more specific packages, run `./dependencies freeze --upgrade-package pkg1 [--upgrade-package pkg2]`"
+        )
+    else:
+        print("Everything appears to be up-to-date!")
+
+
+cli.add_command(outdated)
+
+
+if __name__ == '__main__':
+    cli()

images/hub/requirements.in

@@ -0,0 +1,30 @@
+# JupyterHub's version is defined in chartpress.yaml
+
+## Authenticators
+jupyterhub-dummyauthenticator
+jupyterhub-firstuseauthenticator
+jupyterhub-hmacauthenticator
+jupyterhub-ldapauthenticator
+jupyterhub-ltiauthenticator
+jupyterhub-nativeauthenticator
+jupyterhub-tmpauthenticator
+nullauthenticator
+oauthenticator
+
+# Authenticator optional dependencies
+mwoauth
+globus_sdk[jwt]
+
+## Kubernetes spawner
+jupyterhub-kubespawner
+
+## Other optional dependencies for additional features
+pymysql  # mysql
+psycopg2-binary  # postgres
+pycurl  # internal http requests handle more load with pycurl
+statsd  # statsd metrics collection (TODO: remove soon, since folks use prometheus)
+
+## Useful tools
+
+# py-spy is useful for profiling running hubs
+py-spy

images/hub/requirements.txt

@@ -1,30 +1,77 @@
-# JupyterHub's version is defined in chartpress.yaml
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    ./dependencies freeze
+#
+alembic==1.4.1            # via jupyterhub
+async-generator==1.10     # via jupyterhub, jupyterhub-kubespawner
+attrs==19.3.0             # via jsonschema
+bcrypt==3.1.7             # via jupyterhub-firstuseauthenticator, jupyterhub-nativeauthenticator
+cachetools==4.0.0         # via google-auth
+certifi==2019.11.28       # via kubernetes, requests
+certipy==0.1.3            # via jupyterhub
+cffi==1.14.0              # via bcrypt, cryptography
+chardet==3.0.4            # via requests
+cryptography==2.8         # via pyjwt, pyopenssl
+decorator==4.4.2          # via traitlets
+entrypoints==0.3          # via jupyterhub
+escapism==1.0.0           # via jupyterhub-kubespawner
+globus-sdk[jwt]==1.9.0    # via -r requirements.in
+google-auth==1.11.2       # via kubernetes
+idna==2.9                 # via requests
+importlib-metadata==1.5.0  # via jsonschema
+ipython-genutils==0.2.0   # via traitlets
+jinja2==2.11.1            # via jupyterhub, jupyterhub-kubespawner
+jsonschema==3.2.0         # via jupyter-telemetry
+jupyter-telemetry==0.0.5  # via jupyterhub
+jupyterhub-dummyauthenticator==0.3.1  # via -r requirements.in
+jupyterhub-firstuseauthenticator==0.14.0  # via -r requirements.in
+jupyterhub-hmacauthenticator==0.1  # via -r requirements.in
+jupyterhub-kubespawner==0.11.1  # via -r requirements.in
+jupyterhub-ldapauthenticator==1.3.0  # via -r requirements.in
+jupyterhub-ltiauthenticator==0.4.0  # via -r requirements.in
+jupyterhub-nativeauthenticator==0.0.5  # via -r requirements.in
+jupyterhub-tmpauthenticator==0.6  # via -r requirements.in
+jupyterhub==1.1.0         # via jupyterhub-firstuseauthenticator, jupyterhub-kubespawner, jupyterhub-ldapauthenticator, jupyterhub-ltiauthenticator, jupyterhub-nativeauthenticator, nullauthenticator, oauthenticator
+kubernetes==10.0.1        # via jupyterhub-kubespawner
+ldap3==2.7                # via jupyterhub-ldapauthenticator
+mako==1.1.2               # via alembic
+markupsafe==1.1.1         # via jinja2, mako
+mwoauth==0.3.7            # via -r requirements.in
+nullauthenticator==1.0.0  # via -r requirements.in
+oauthenticator==0.11.0    # via -r requirements.in
+oauthlib==3.1.0           # via jupyterhub, jupyterhub-ltiauthenticator, mwoauth, requests-oauthlib
+onetimepass==1.0.1        # via jupyterhub-nativeauthenticator
+pamela==1.0.0             # via jupyterhub
+prometheus-client==0.7.1  # via jupyterhub
+psycopg2-binary==2.8.4    # via -r requirements.in
+py-spy==0.3.3             # via -r requirements.in
+pyasn1-modules==0.2.8     # via google-auth
+pyasn1==0.4.8             # via ldap3, pyasn1-modules, rsa
+pycparser==2.20           # via cffi
+pycurl==7.43.0.5          # via -r requirements.in
+pyjwt[crypto]==1.7.1      # via globus-sdk, mwoauth
+pymysql==0.9.3            # via -r requirements.in
+pyopenssl==19.1.0         # via certipy
+pyrsistent==0.15.7        # via jsonschema
+python-dateutil==2.8.1    # via alembic, jupyterhub, kubernetes
+python-editor==1.0.4      # via alembic
+python-json-logger==0.1.11  # via jupyter-telemetry
+pyyaml==5.3               # via jupyterhub-kubespawner, kubernetes
+requests-oauthlib==1.3.0  # via kubernetes, mwoauth
+requests==2.23.0          # via globus-sdk, jupyterhub, kubernetes, mwoauth, requests-oauthlib
+rsa==4.0                  # via google-auth
+ruamel.yaml.clib==0.2.0   # via ruamel.yaml
+ruamel.yaml==0.16.10      # via jupyter-telemetry
+six==1.14.0               # via bcrypt, cryptography, globus-sdk, google-auth, jsonschema, kubernetes, mwoauth, onetimepass, pyopenssl, pyrsistent, python-dateutil, traitlets, websocket-client
+sqlalchemy==1.3.15        # via alembic, jupyterhub
+statsd==3.3.0             # via -r requirements.in
+tornado==6.0.4            # via jupyterhub, jupyterhub-ldapauthenticator
+traitlets==4.3.3          # via jupyter-telemetry, jupyterhub, jupyterhub-ldapauthenticator
+urllib3==1.25.8           # via kubernetes, requests
+websocket-client==0.57.0  # via kubernetes
+zipp==3.1.0               # via importlib-metadata
 
-## Authenticators
-jupyterhub-dummyauthenticator==0.3.*
-jupyterhub-firstuseauthenticator==0.12.*
-jupyterhub-tmpauthenticator==0.6.*
-jupyterhub-ltiauthenticator==0.4.*
-jupyterhub-ldapauthenticator==1.3.*
-jupyterhub-hmacauthenticator==0.1.*
-jupyterhub-nativeauthenticator==0.0.5
-mwoauth==0.3.7
-globus_sdk[jwt]==1.8.*
-nullauthenticator==1.0.*
-oauthenticator==0.11.*
-
-## Spawners
-jupyterhub-kubespawner==0.11.*
-kubernetes==10.0.*
-
-## Other dependencies
-pymysql==0.9.*
-psycopg2-binary==2.8.*
-pycurl==7.43.0.*
-statsd==3.3.*
-cryptography==2.8.*
-
-## Useful tools
-
-# py-spy is useful for profiling running hubs
-py-spy
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools