General maintenance: autoformat, flake8, readme badges, misc in github workflows

.flake8

@@ -0,0 +1,13 @@
+[flake8]
+# Ignore style and complexity
+# E: style errors
+# W: style warnings
+# C: complexity
+# F401: module imported but unused
+# F403: import *
+# F811: redefinition of unused `name` from line `N`
+# F841: local variable assigned but never used
+# E402: module level import not at top of file
+# I100: Import statements are in the wrong order
+# I101: Imported names are in the wrong order. Should be
+ignore = E, W, C, F401, F403, F811, F841, E402, I100, I101, D400

.github/workflows/python-publish.yml

@@ -9,23 +9,22 @@ on:
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python
-      uses: actions/setup-python@v2
-      with:
-        python-version: '3.x'
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: '__token__'
-        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+      - uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.x"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install setuptools wheel twine
+      - name: Build and publish
+        env:
+          TWINE_USERNAME: "__token__"
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+        run: |
+          python setup.py sdist bdist_wheel
+          twine upload dist/*

.github/workflows/test.yml

@@ -1,55 +1,77 @@
+# This is a GitHub workflow defining a set of jobs with a set of steps.
+# ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
+#
+name: Test
 
-name: Python package
-
-on: [push, pull_request]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 
 jobs:
-  build:
+  pre-commit:
+    name: Run pre-commit
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+      - uses: pre-commit/action@v2.0.0
 
+  pytest:
+    name: "Run pytest"
     runs-on: ubuntu-20.04
     continue-on-error: ${{ matrix.allow_failure }}
     strategy:
+      # Keep running even if one variation of the job fail
+      fail-fast: false
       matrix:
         python-version:
-          - '3.5'
-          - '3.9'
+          - "3.5"
+          - "3.9"
         JHUB_VER:
-          - '0.9.6'
-          - '1.0.0'
-          - '1.1.0'
-          - '1.2.0'
+          - "0.9.6"
+          - "1.0.0"
+          - "1.1.0"
+          - "1.2.0"
+          - "1.3.0"
         allow_failure: [false]
+
+        exclude:
+          # JupyterHub 1.3.0 requires python 3.6+
+          - JHUB_VER: "1.3.0"
+            python-version: "3.5"
         include:
-          - JHUB_VER: 'master'
-            python-version: '3.9'
+          - JHUB_VER: "master"
+            python-version: "3.9"
             allow_failure: true
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        python -m pip install pytest
-        pip install -r requirements.txt
-    - name: Install nodejs dependencies
-      run: |
-        sudo npm install -g configurable-http-proxy
-    # We need to check compatibility with different versions of the JH
-    # API, including latest development.  For that, we also need to
-    # pull in the dependencies of that old JH version (but we don't
-    # need conda/npm for our tests).
-    - name: install JupyterHub
-      run: |
-        git clone --quiet --branch ${{ matrix.JHUB_VER }} https://github.com/jupyterhub/jupyterhub.git jupyterhub
-        pip install --pre -r jupyterhub/dev-requirements.txt
-        pip install --upgrade pytest
-        pip install --pre -e jupyterhub
-
-    - name: pytest
-      run: |
-        pytest --lf --cov batchspawner batchspawner/tests -v
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install pytest
+          pip install -r requirements.txt
+      - name: Install nodejs dependencies
+        run: |
+          sudo npm install -g configurable-http-proxy
+
+      # We need to check compatibility with different versions of the JH
+      # API, including latest development.  For that, we also need to
+      # pull in the dependencies of that old JH version (but we don't
+      # need conda/npm for our tests).
+      - name: install JupyterHub
+        run: |
+          git clone --quiet --branch ${{ matrix.JHUB_VER }} https://github.com/jupyterhub/jupyterhub.git jupyterhub
+          pip install --pre -r jupyterhub/dev-requirements.txt
+          pip install --upgrade pytest
+          pip install --pre -e jupyterhub
+
+      - name: pytest
+        run: |
+          pytest --verbose --color=yes --last-failed --cov batchspawner batchspawner/tests

.pre-commit-config.yaml

@@ -0,0 +1,50 @@
+# pre-commit is a tool to perform a predefined set of tasks manually and/or
+# automatically before git commits are made.
+#
+# Config reference: https://pre-commit.com/#pre-commit-configyaml---top-level
+#
+# Common tasks
+#
+# - Run on all files:   pre-commit run --all-files
+# - Register git hooks: pre-commit install --install-hooks
+#
+repos:
+  # Autoformat: Python code
+  - repo: https://github.com/ambv/black
+    rev: 20.8b1
+    hooks:
+      - id: black
+        args: [--target-version=py36]
+
+  # Autoformat: markdown, yaml
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v2.2.1
+    hooks:
+      - id: prettier
+
+  # Lint: Python code
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: "3.8.4"
+    hooks:
+      - id: flake8
+
+  # Misc...
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.4.0
+    # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available
+    hooks:
+      # Autoformat: Makes sure files end in a newline and only a newline.
+      - id: end-of-file-fixer
+
+      # Autoformat: Sorts entries in requirements.txt.
+      - id: requirements-txt-fixer
+
+      # Prevent giant (500kB) files from being committed.
+      - id: check-added-large-files
+
+      # Lint: Check for files with names that would conflict on a
+      # case-insensitive filesystem like MacOS HFS+ or Windows FAT.
+      - id: check-case-conflict
+
+      # Lint: Checks that non-binary executables have a proper shebang.
+      - id: check-executables-have-shebangs

CHANGELOG.md

@@ -14,85 +14,81 @@ Fixed
 
 Added (user)
 
-* PR #170: SlurmSpawner: add `req_gres` to specify `-go-res`.
-* PR #137: GridEngineSpawner: spawner will now add the following system environment values to the spawner environment, in accordance with the Univa Admin Guide: `SGE_CELL`, `SGE_EXECD`, `SGE_ROOT`, `SGE_CLUSTER_NAME`, `SGE_QMASTER_PORT`, `SGE_EXECD_PORT`, `PATH`
+- PR #170: SlurmSpawner: add `req_gres` to specify `-go-res`.
+- PR #137: GridEngineSpawner: spawner will now add the following system environment values to the spawner environment, in accordance with the Univa Admin Guide: `SGE_CELL`, `SGE_EXECD`, `SGE_ROOT`, `SGE_CLUSTER_NAME`, `SGE_QMASTER_PORT`, `SGE_EXECD_PORT`, `PATH`
 
 Added (developer)
 
-* PR #187: support for unknown job state
+- PR #187: support for unknown job state
 
 Changed
 
-* PR #177: Fail on first error in batch script by setting `set -e` to script templates.
-* PR #165: SlurmSpawner: Update template to use `--chdir` instead of `--workdir`. Users of Slurm older than 17.11 may need to revert this locally.
-* PR #189: remove bashism from default script template
-* PR #195: fix exception handling in run_command
-* PR #198: change from Travis to gh-actions for testing
-* PR #196: documentation
-* PR #199: update setup.py
+- PR #177: Fail on first error in batch script by setting `set -e` to script templates.
+- PR #165: SlurmSpawner: Update template to use `--chdir` instead of `--workdir`. Users of Slurm older than 17.11 may need to revert this locally.
+- PR #189: remove bashism from default script template
+- PR #195: fix exception handling in run_command
+- PR #198: change from Travis to gh-actions for testing
+- PR #196: documentation
+- PR #199: update setup.py
 
 ## v1.0 (requires minimum JupyterHub 0.9 and Python 3.5)
 
 Added (user)
 
-* Add support for JupyterHub named servers. #167
-* Add Jinja2 templating as an option for all scripts and commands.  If '{{' or `{%` is used anywhere in the string, it is used as a jinja2 template.
-* Add new option exec_prefix, which defaults to `sudo -E -u {username}`.  This replaces explicit `sudo` in every batch command - changes in local commands may be needed.
-* New option: `req_keepvars_extra`, which allows keeping extra variables in addition to what is defined by JupyterHub itself (addition of variables to keep instead of replacement).  #99
-* Add `req_prologue` and `req_epilogue` options to scripts which are inserted before/after the main jupyterhub-singleuser command, which allow for generic setup/cleanup without overriding the entire script.  #96
-* SlurmSpawner: add the `req_reservation` option.  #91
-* Add basic support for JupyterHub progress updates, but this is not used much yet.  #86
+- Add support for JupyterHub named servers. #167
+- Add Jinja2 templating as an option for all scripts and commands. If '{{' or `{%` is used anywhere in the string, it is used as a jinja2 template.
+- Add new option exec_prefix, which defaults to `sudo -E -u {username}`. This replaces explicit `sudo` in every batch command - changes in local commands may be needed.
+- New option: `req_keepvars_extra`, which allows keeping extra variables in addition to what is defined by JupyterHub itself (addition of variables to keep instead of replacement). #99
+- Add `req_prologue` and `req_epilogue` options to scripts which are inserted before/after the main jupyterhub-singleuser command, which allow for generic setup/cleanup without overriding the entire script. #96
+- SlurmSpawner: add the `req_reservation` option. #91
+- Add basic support for JupyterHub progress updates, but this is not used much yet. #86
 
 Added (developer)
 
-* Add many more tests.
-* Add a new page `SPAWNERS.md` which information on specific spawners.  Begin trying to collect a list of spawner-specific contacts.  #97
-* Rename `current_ip` and `current_port` commands to `ip` and `port`.  No user impact.  #139
-* Update to Python 3.5 `async` / `await` syntax to support JupyterHub progress updates.  #90
+- Add many more tests.
+- Add a new page `SPAWNERS.md` which information on specific spawners. Begin trying to collect a list of spawner-specific contacts. #97
+- Rename `current_ip` and `current_port` commands to `ip` and `port`. No user impact. #139
+- Update to Python 3.5 `async` / `await` syntax to support JupyterHub progress updates. #90
 
 Changed
 
-* PR #58 and #141 changes logic of port selection, so that it is selected *after* the singleuser server starts.  This means that the port number has to be conveyed back to JupyterHub.  This requires the following changes:
-  - `jupyterhub_config.py` *must* explicitely import `batchspawner`
-  - Add a new option `batchspawner_singleuser_cmd` which is used as a wrapper in the single-user servers, which conveys the remote port back to JupyterHub.  This is now an integral part of the spawn process.
+- PR #58 and #141 changes logic of port selection, so that it is selected _after_ the singleuser server starts. This means that the port number has to be conveyed back to JupyterHub. This requires the following changes:
+  - `jupyterhub_config.py` _must_ explicitely import `batchspawner`
+  - Add a new option `batchspawner_singleuser_cmd` which is used as a wrapper in the single-user servers, which conveys the remote port back to JupyterHub. This is now an integral part of the spawn process.
   - If you have installed with `pip install -e`, you will have to re-install so that the new script `batchspawner-singleuser` is added to `$PATH`.
-* Update minimum requirements to JupyterHub 0.9 and Python 3.5.  #143
-* Update Slurm batch script.  Now, the single-user notebook is run in a job step, with a wrapper of `srun`.  This may need to be removed using `req_srun=''` if you don't want environment variables limited.
-* Pass the environment dictionary to the queue and cancel commands as well.  This is mostly user environment, but may be useful to these commands as well in some cases. #108, #111  If these environment variables were used for authentication as an admin, be aware that there are pre-existing security issues because they may be passed to the user via the batch submit command, see #82.
-
+- Update minimum requirements to JupyterHub 0.9 and Python 3.5. #143
+- Update Slurm batch script. Now, the single-user notebook is run in a job step, with a wrapper of `srun`. This may need to be removed using `req_srun=''` if you don't want environment variables limited.
+- Pass the environment dictionary to the queue and cancel commands as well. This is mostly user environment, but may be useful to these commands as well in some cases. #108, #111 If these environment variables were used for authentication as an admin, be aware that there are pre-existing security issues because they may be passed to the user via the batch submit command, see #82.
 
 Fixed
 
-* Improve debugging on failed submission by raising errors including error messages from the commands.  #106
-* Many other non-user or developer visible changes.  #107 #106 #100
-* In Travis CI, blacklist jsonschema=3.0.0a1 because it breaks tests
+- Improve debugging on failed submission by raising errors including error messages from the commands. #106
+- Many other non-user or developer visible changes. #107 #106 #100
+- In Travis CI, blacklist jsonschema=3.0.0a1 because it breaks tests
 
 Removed
 
-
 ## v0.8.1 (bugfix release)
 
-* Fix regression: single-user server binding address is overwritten by previous session server address, resulting in failure to start.  Issue #76
+- Fix regression: single-user server binding address is overwritten by previous session server address, resulting in failure to start. Issue #76
 
 ## v0.8.0 (compatible with JupyterHub 0.5.0 through 0.8.1/0.9dev)
 
-* SlurmSpawner: Remove `--uid` for (at least) Slurm 17.11 compatibility.  If you use `sudo`, this should not be necessary, but because this is security related you should check that user management is as you expect.  If your configuration does not use `sudo` then you may need to add the `--uid` option in a custom `batch_script`.
-* add base options `req_ngpus` `req_partition` `req_account` and `req_options` 
-* Fix up logging
-* Merge `user_options` with the template substitution vars instead of having it as a separate key
-* Update ip/port handling for JupyterHub 0.8
-* Add `LICENSE` (BSD3) and `CONTRIBUTING.md`
-* Add `LsfSpawner` for IBM LFS
-* Add `MultiSlurmSpawner`
-* Add `MoabSpawner`
-* Add `condorSpawner`
-* Add `GridEngineSpawner`
-* SlurmSpawner: add `req_qos` option
-* WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners, have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package
-* Enable CI testing via Travis-CI
-
+- SlurmSpawner: Remove `--uid` for (at least) Slurm 17.11 compatibility. If you use `sudo`, this should not be necessary, but because this is security related you should check that user management is as you expect. If your configuration does not use `sudo` then you may need to add the `--uid` option in a custom `batch_script`.
+- add base options `req_ngpus` `req_partition` `req_account` and `req_options`
+- Fix up logging
+- Merge `user_options` with the template substitution vars instead of having it as a separate key
+- Update ip/port handling for JupyterHub 0.8
+- Add `LICENSE` (BSD3) and `CONTRIBUTING.md`
+- Add `LsfSpawner` for IBM LFS
+- Add `MultiSlurmSpawner`
+- Add `MoabSpawner`
+- Add `condorSpawner`
+- Add `GridEngineSpawner`
+- SlurmSpawner: add `req_qos` option
+- WrapSpawner and ProfilesSpawner, which provide mechanisms for runtime configuration of spawners, have been split out and moved to the [`wrapspawner`](https://github.com/jupyterhub/wrapspawner) package
+- Enable CI testing via Travis-CI
 
 ## v0.3 (tag: jhub-0.3, compatible with JupyterHub 0.3.0)
 
-* initial release containing `TorqueSpawner` and `SlurmSpawner`
-
+- initial release containing `TorqueSpawner` and `SlurmSpawner`