diff --git a/dev-requirements.txt b/dev-requirements.txt index 4b6958c5f0..fc70e6b6d2 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -24,3 +24,6 @@ yamllint>=1.17.0 ## ./dev use these colorama python-dotenv + +# click is used to make cli scripts +click diff --git a/images/hub/Dockerfile b/images/hub/Dockerfile index 68742a1919..4550932928 100644 --- a/images/hub/Dockerfile +++ b/images/hub/Dockerfile @@ -40,7 +40,8 @@ RUN adduser --disabled-password \ --force-badname \ ${NB_USER} -ADD requirements.txt /tmp/requirements.txt +RUN python3 -m pip install --upgrade --no-cache setuptools pip +COPY requirements.txt /tmp/requirements.txt RUN PYCURL_SSL_LIBRARY=openssl pip3 install --no-cache-dir \ -r /tmp/requirements.txt \ $(bash -c 'if [[ $JUPYTERHUB_VERSION == "git"* ]]; then \ @@ -57,5 +58,12 @@ RUN chown ${NB_USER}:${NB_USER} /srv/jupyterhub # JupyterHub API port EXPOSE 8081 +# when building the dependencies image +# add pip-tools necessary for computing dependencies +# this is not done in production builds by chartpress +ARG PIP_TOOLS= +RUN test -z "$PIP_TOOLS" || pip install --no-cache pip-tools==$PIP_TOOLS + USER ${NB_USER} CMD ["jupyterhub", "--config", "/etc/jupyterhub/jupyterhub_config.py"] + diff --git a/images/hub/dependencies b/images/hub/dependencies new file mode 100755 index 0000000000..0a436a93fa --- /dev/null +++ b/images/hub/dependencies @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""automatically manage requirements.txt dependencies with pip-tools + +See + + ./dependencies --help for commands and arguments + +How it works: + +- the image used in the helm chart installs a frozen environment from requirements.txt +- `pip-compile` is used to generate frozen `requirements.txt` from our actual requirements + in requirements.in. +- `pip list --outdated` is used to report available updates for packages + in the frozen environment +- pip-compile etc. are run *inside the image* to ensure consistent behavior, + rather than running on host systems, which can vary. +- When building the image to be used for running dependency-management commands, + chartpress configuration is loaded to ensure the environment is the same + as when chartpress builds the tagged image to be published. +""" + +from functools import lru_cache +import json +import os +from subprocess import check_call, check_output + +import click +from ruamel.yaml import YAML + +yaml = YAML() +here = os.path.dirname(os.path.abspath(__file__)) +chartpress_yaml = os.path.join(here, os.pardir, os.pardir, "chartpress.yaml") +values_yaml = os.path.join(here, os.pardir, os.pardir, "jupyterhub", "values.yaml") +dependencies_image = 'hub-dependencies' +pip_tools_version="4.*" + + +@lru_cache() +def build_args(image_name='hub'): + """retrieve docker build arguments from chartpress.yaml config file + + Args: + + image_name (str): + the name of the image to be built in chartpress.yaml + """ + with open(chartpress_yaml) as f: + chartpress_config = yaml.load(f) + chart = chartpress_config['charts'][0] + image_config = chart['images'][image_name] + return image_config.get('buildArgs', {}) + + +def build_image(): + """Build the docker image used for computing dependencies + + This runs the chartpress build of the current image + with the addition of pip-tools, used for computing dependencies. + + The image is built with the current frozen environment in requirements.txt + and pip-tools commands are available for updating requirements.txt from requirements.in. + """ + click.echo(f"Building docker image {dependencies_image}") + build_arg_dict = build_args() + build_arg_list = ["--build-arg", f"PIP_TOOLS={pip_tools_version}"] + for key in sorted(build_arg_dict): + value = build_arg_dict[key] + build_arg_list.append("--build-arg") + build_arg_list.append(f"{key}={value}") + check_call(["docker", "build", "-t", dependencies_image] + build_arg_list + [here]) + + +@click.group() +def cli(): + """Manage the Python dependencies in this image.""" + pass + + +@click.command() +@click.option( + '--build/--no-build', + help="add --no-build to skip building the dependencies image prior to upgrading", + default=True, +) +@click.option( + '--upgrade/--no-upgrade', + help="--upgrade to upgrade all dependencies within the range specified in requirements.in", + default=False, +) +@click.option( + '--upgrade-package', + help="specify individual packages to upgrade within the range specified in requirements.in", + multiple=True, +) +def freeze(build, upgrade, upgrade_package): + """Freeze the environment, updating requirements.txt from requirements.in + + Individual packages can be updated, or the whole environment. + + This command: + + 1. builds the image with the current frozen environment + 2. runs pip-compile in the image to update requirements.txt from requirements.in, + passing through additional arguments to pip-compile + """ + if build: + build_image() + click.echo("freezing dependencies with pip-compile") + upgrade_args = [] + if upgrade: + upgrade_args.append("--upgrade") + for pkg in upgrade_package: + upgrade_args.append("--upgrade-package") + upgrade_args.append(pkg) + check_call( + [ + "docker", + "run", + "--rm", + "-it", + "-e", + "CUSTOM_COMPILE_COMMAND=./dependencies freeze", + "--volume", + f"{here}:/io", + "--workdir", + "/io", + dependencies_image, + "pip-compile", + ] + + upgrade_args + ) + + +cli.add_command(freeze) + + +@click.command() +@click.option('--build/--no-build', default=True) +def outdated(build): + """Check for outdated dependencies with pip. + + This command: + + 1. builds the image with the current frozen environment + 2. runs `pip list --outdated` to report any outdated packages + that could be candidates for upgrade + """ + if build: + build_image() + click.echo("Checking for outdated dependencies with pip.") + outdated_json = check_output( + [ + "docker", + "run", + "--rm", + "-it", + dependencies_image, + "pip", + "list", + "--outdated", + "--format=json", + ] + ).decode("utf8") + outdated = json.loads(outdated_json) + have_outdated = False + for pkg in outdated: + name = pkg['name'] + # ignore some common packages that aren't relevant to our requirements.txt + if name in {'pip', 'setuptools', 'wheel'}: + continue + have_outdated = True + version = pkg['version'] + latest = pkg['latest_version'] + # TODO: parser requirements.in to check if latest is in-range? + # If they are in-range, running freeze again is enough, + # but if they are outside the range, requirements.in needs to be updated + # first to pick them up + # for now, print as much so humans can decide + print(f"Have {name}=={version}, latest is {name}=={latest}") + + if have_outdated: + print("There are outdated dependencies!") + print( + "To pick up any versions outside the range(s) specified in requirements.in," + ) + print("update the pinning(s) in that file.") + print( + "To update the whole environment within the given ranges, run `./dependencies freeze --upgrade`" + ) + print( + "To update one or more specific packages, run `./dependencies freeze --upgrade-package pkg1 [--upgrade-package pkg2]`" + ) + else: + print("Everything appears to be up-to-date!") + + +cli.add_command(outdated) + + +if __name__ == '__main__': + cli() diff --git a/images/hub/requirements.in b/images/hub/requirements.in new file mode 100644 index 0000000000..5de46f3315 --- /dev/null +++ b/images/hub/requirements.in @@ -0,0 +1,30 @@ +# JupyterHub's version is defined in chartpress.yaml + +## Authenticators +jupyterhub-dummyauthenticator +jupyterhub-firstuseauthenticator +jupyterhub-hmacauthenticator +jupyterhub-ldapauthenticator +jupyterhub-ltiauthenticator +jupyterhub-nativeauthenticator +jupyterhub-tmpauthenticator +nullauthenticator +oauthenticator + +# Authenticator optional dependencies +mwoauth +globus_sdk[jwt] + +## Kubernetes spawner +jupyterhub-kubespawner + +## Other optional dependencies for additional features +pymysql # mysql +psycopg2-binary # postgres +pycurl # internal http requests handle more load with pycurl +statsd # statsd metrics collection (TODO: remove soon, since folks use prometheus) + +## Useful tools + +# py-spy is useful for profiling running hubs +py-spy diff --git a/images/hub/requirements.txt b/images/hub/requirements.txt index 7f7b0a81d5..22f0b609e7 100644 --- a/images/hub/requirements.txt +++ b/images/hub/requirements.txt @@ -1,30 +1,77 @@ -# JupyterHub's version is defined in chartpress.yaml +# +# This file is autogenerated by pip-compile +# To update, run: +# +# ./dependencies freeze +# +alembic==1.4.1 # via jupyterhub +async-generator==1.10 # via jupyterhub, jupyterhub-kubespawner +attrs==19.3.0 # via jsonschema +bcrypt==3.1.7 # via jupyterhub-firstuseauthenticator, jupyterhub-nativeauthenticator +cachetools==4.0.0 # via google-auth +certifi==2019.11.28 # via kubernetes, requests +certipy==0.1.3 # via jupyterhub +cffi==1.14.0 # via bcrypt, cryptography +chardet==3.0.4 # via requests +cryptography==2.8 # via pyjwt, pyopenssl +decorator==4.4.2 # via traitlets +entrypoints==0.3 # via jupyterhub +escapism==1.0.0 # via jupyterhub-kubespawner +globus-sdk[jwt]==1.9.0 # via -r requirements.in +google-auth==1.11.2 # via kubernetes +idna==2.9 # via requests +importlib-metadata==1.5.0 # via jsonschema +ipython-genutils==0.2.0 # via traitlets +jinja2==2.11.1 # via jupyterhub, jupyterhub-kubespawner +jsonschema==3.2.0 # via jupyter-telemetry +jupyter-telemetry==0.0.5 # via jupyterhub +jupyterhub-dummyauthenticator==0.3.1 # via -r requirements.in +jupyterhub-firstuseauthenticator==0.14.0 # via -r requirements.in +jupyterhub-hmacauthenticator==0.1 # via -r requirements.in +jupyterhub-kubespawner==0.11.1 # via -r requirements.in +jupyterhub-ldapauthenticator==1.3.0 # via -r requirements.in +jupyterhub-ltiauthenticator==0.4.0 # via -r requirements.in +jupyterhub-nativeauthenticator==0.0.5 # via -r requirements.in +jupyterhub-tmpauthenticator==0.6 # via -r requirements.in +jupyterhub==1.1.0 # via jupyterhub-firstuseauthenticator, jupyterhub-kubespawner, jupyterhub-ldapauthenticator, jupyterhub-ltiauthenticator, jupyterhub-nativeauthenticator, nullauthenticator, oauthenticator +kubernetes==10.0.1 # via jupyterhub-kubespawner +ldap3==2.7 # via jupyterhub-ldapauthenticator +mako==1.1.2 # via alembic +markupsafe==1.1.1 # via jinja2, mako +mwoauth==0.3.7 # via -r requirements.in +nullauthenticator==1.0.0 # via -r requirements.in +oauthenticator==0.11.0 # via -r requirements.in +oauthlib==3.1.0 # via jupyterhub, jupyterhub-ltiauthenticator, mwoauth, requests-oauthlib +onetimepass==1.0.1 # via jupyterhub-nativeauthenticator +pamela==1.0.0 # via jupyterhub +prometheus-client==0.7.1 # via jupyterhub +psycopg2-binary==2.8.4 # via -r requirements.in +py-spy==0.3.3 # via -r requirements.in +pyasn1-modules==0.2.8 # via google-auth +pyasn1==0.4.8 # via ldap3, pyasn1-modules, rsa +pycparser==2.20 # via cffi +pycurl==7.43.0.5 # via -r requirements.in +pyjwt[crypto]==1.7.1 # via globus-sdk, mwoauth +pymysql==0.9.3 # via -r requirements.in +pyopenssl==19.1.0 # via certipy +pyrsistent==0.15.7 # via jsonschema +python-dateutil==2.8.1 # via alembic, jupyterhub, kubernetes +python-editor==1.0.4 # via alembic +python-json-logger==0.1.11 # via jupyter-telemetry +pyyaml==5.3 # via jupyterhub-kubespawner, kubernetes +requests-oauthlib==1.3.0 # via kubernetes, mwoauth +requests==2.23.0 # via globus-sdk, jupyterhub, kubernetes, mwoauth, requests-oauthlib +rsa==4.0 # via google-auth +ruamel.yaml.clib==0.2.0 # via ruamel.yaml +ruamel.yaml==0.16.10 # via jupyter-telemetry +six==1.14.0 # via bcrypt, cryptography, globus-sdk, google-auth, jsonschema, kubernetes, mwoauth, onetimepass, pyopenssl, pyrsistent, python-dateutil, traitlets, websocket-client +sqlalchemy==1.3.15 # via alembic, jupyterhub +statsd==3.3.0 # via -r requirements.in +tornado==6.0.4 # via jupyterhub, jupyterhub-ldapauthenticator +traitlets==4.3.3 # via jupyter-telemetry, jupyterhub, jupyterhub-ldapauthenticator +urllib3==1.25.8 # via kubernetes, requests +websocket-client==0.57.0 # via kubernetes +zipp==3.1.0 # via importlib-metadata -## Authenticators -jupyterhub-dummyauthenticator==0.3.* -jupyterhub-firstuseauthenticator==0.12.* -jupyterhub-tmpauthenticator==0.6.* -jupyterhub-ltiauthenticator==0.4.* -jupyterhub-ldapauthenticator==1.3.* -jupyterhub-hmacauthenticator==0.1.* -jupyterhub-nativeauthenticator==0.0.5 -mwoauth==0.3.7 -globus_sdk[jwt]==1.8.* -nullauthenticator==1.0.* -oauthenticator==0.11.* - -## Spawners -jupyterhub-kubespawner==0.11.* -kubernetes==10.0.* - -## Other dependencies -pymysql==0.9.* -psycopg2-binary==2.8.* -pycurl==7.43.0.* -statsd==3.3.* -cryptography==2.8.* - -## Useful tools - -# py-spy is useful for profiling running hubs -py-spy +# The following packages are considered to be unsafe in a requirements file: +# setuptools