Build 3.12 (#733)

* Update builds for Python 3.11

* Adding initial 3.12 support

* Added 3.12 workarounds

* CI fix attempts

* Fix docs build

* Added tox installer script in manifest

* Removed ipynb checkpoints from manifest

* Fixed tox build for docs

* Build fixes

* Disabled frozen debug warning in CLI

* Added docs requirements to manifest

* Removed tar build test from py312 build flow

* Removed doc references to 3.7

---------

Co-authored-by: Karthikeyan Singaravelan <tir.karthi@gmail.com>

.github/workflows/ci.yml

@@ -12,12 +12,12 @@ jobs:
     name: "Python ${{ matrix.python-version }}"
     runs-on: "ubuntu-latest"
     env:
-      USING_COVERAGE: '3.8,3.9,3.10'
+      USING_COVERAGE: '3.8,3.9,3.10,3.11,3.12'
 
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - uses: "actions/checkout@v3"
@@ -54,7 +54,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.12
       - name: "Install dependencies"
         run: |
           set -xe

CONTRIBUTING.md

@@ -69,7 +69,7 @@ For a full test suite of all envs and linting checks simply run tox without any
 tox
 ```
 
-This will require python3.7, python3.8, python3.9 and python3.10 to be installed. **Note** that python 3.7 has problems with the alpha build which is the available package version on many linux distros. Local build failures with 3.7 can happen as a result (you'll see a seg fault or exist code -11).
+This will require python3.8, 3.9, 3.10, 3.11, and 3.12 to be installed.
 
 Alternavitely pytest can be used if you have an environment already setup which works or has custom packages not present in the tox build.
 

MANIFEST.in

@@ -8,7 +8,9 @@ recursive-include papermill *.txt
 
 include setup.py
 include requirements.txt
+include tox_py_installer
 recursive-include requirements *.txt
+include docs/requirements.txt
 include tox.ini
 include pytest.ini
 include README.md
@@ -29,6 +31,9 @@ prune binder
 graft scripts
 # Test env
 prune .tox
+# Exclude notebooks checkpoints generated by testing
+recursive-exclude papermill/.ipynb_checkpoints *.ipynb
+recursive-exclude papermill/tests/notebooks/.ipynb_checkpoints *.ipynb
 
 # Build files
 exclude .github

README.md

@@ -54,7 +54,7 @@ pip install papermill[all]
 
 ## Python Version Support
 
-This library currently supports Python 3.7+ versions. As minor Python
+This library currently supports Python 3.8+ versions. As minor Python
 versions are officially sunset by the Python org papermill will similarly
 drop support in the future.
 

binder/cli-simple/simple_output.ipynb

@@ -2,12 +2,13 @@
  "cells": [
   {
    "cell_type": "markdown",
+   "id": "df2b0bde",
    "metadata": {
     "papermill": {
-     "duration": 0.00964,
-     "end_time": "2020-08-17T19:57:24.239226",
+     "duration": 0.003083,
+     "end_time": "2023-11-01T18:34:41.235066",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.229586",
+     "start_time": "2023-11-01T18:34:41.231983",
      "status": "completed"
     },
     "tags": []
@@ -19,18 +20,19 @@
   {
    "cell_type": "code",
    "execution_count": 1,
+   "id": "0d7af2f3",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.263404Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.262659Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.265395Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.264689Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.301099Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.300307Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.306171Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.305038Z"
     },
     "papermill": {
-     "duration": 0.012719,
-     "end_time": "2020-08-17T19:57:24.265567",
+     "duration": 0.012465,
+     "end_time": "2023-11-01T18:34:41.308519",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.252848",
+     "start_time": "2023-11-01T18:34:41.296054",
      "status": "completed"
     },
     "tags": [
@@ -45,18 +47,19 @@
   {
    "cell_type": "code",
    "execution_count": 2,
+   "id": "a5d4ad35",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.274405Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.273934Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.276194Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.275695Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.314681Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.314338Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.318662Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.317802Z"
     },
     "papermill": {
-     "duration": 0.007713,
-     "end_time": "2020-08-17T19:57:24.276300",
+     "duration": 0.010058,
+     "end_time": "2023-11-01T18:34:41.320883",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.268587",
+     "start_time": "2023-11-01T18:34:41.310825",
      "status": "completed"
     },
     "tags": [
@@ -72,18 +75,19 @@
   {
    "cell_type": "code",
    "execution_count": 3,
+   "id": "99512b79",
    "metadata": {
     "execution": {
-     "iopub.execute_input": "2020-08-17T19:57:24.283119Z",
-     "iopub.status.busy": "2020-08-17T19:57:24.282711Z",
-     "iopub.status.idle": "2020-08-17T19:57:24.284846Z",
-     "shell.execute_reply": "2020-08-17T19:57:24.284482Z"
+     "iopub.execute_input": "2023-11-01T18:34:41.327359Z",
+     "iopub.status.busy": "2023-11-01T18:34:41.326530Z",
+     "iopub.status.idle": "2023-11-01T18:34:41.333035Z",
+     "shell.execute_reply": "2023-11-01T18:34:41.331879Z"
     },
     "papermill": {
-     "duration": 0.006366,
-     "end_time": "2020-08-17T19:57:24.284929",
+     "duration": 0.012117,
+     "end_time": "2023-11-01T18:34:41.335151",
      "exception": false,
-     "start_time": "2020-08-17T19:57:24.278563",
+     "start_time": "2023-11-01T18:34:41.323034",
      "status": "completed"
     },
     "tags": []
@@ -120,22 +124,23 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.12.0"
   },
   "papermill": {
-   "duration": 0.727728,
-   "end_time": "2020-08-17T19:57:24.494284",
+   "default_parameters": {},
+   "duration": 1.612177,
+   "end_time": "2023-11-01T18:34:41.659543",
    "environment_variables": {},
    "exception": null,
    "input_path": "binder/cli-simple/simple_input.ipynb",
    "output_path": "binder/cli-simple/simple_output.ipynb",
    "parameters": {
     "msg": "Hello"
    },
-   "start_time": "2020-08-17T19:57:23.766556",
-   "version": "2.1.3"
+   "start_time": "2023-11-01T18:34:40.047366",
+   "version": "2.4.0"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 1
-}
+ "nbformat_minor": 5
+}

docs/changelog.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Added support for python 3.11 and 3.12
+
 ## 2.4.0
 
 - Add tracking cell executions with cell descriptions [PR #650](https://github.com/nteract/papermill/pull/650)

docs/conf.py

@@ -73,7 +73,7 @@
 #
 # This is also used if you do content translation via gettext catalogs.
 # Usually you set "language" from the command line foexitr these cases.
-language = None
+language = 'python'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -163,4 +163,4 @@
 ]
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {'python': ('https://docs.python.org/', None)}

docs/index.rst

@@ -34,7 +34,7 @@ This opens up new opportunities for how notebooks can be used. For example:
 Python Version Support
 ----------------------
 
-This library currently supports python 3.7+ versions. As minor python
+This library currently supports python 3.8+ versions. As minor python
 versions are officially sunset by the python org papermill will similarly
 drop support in the future.
 

docs/requirements.txt

@@ -1,8 +1,8 @@
 # Pin packages for RTD builds
 # https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html#pinning-dependencies
--r ../requirements.txt
-Sphinx==3.5.4
-furo==2021.4.11b34
-myst-parser==0.13.7
-moto==2.0.5
-sphinx-copybutton==0.3.1
+#-r ../requirements.txt
+Sphinx>=3.5.4
+furo>=2021.4.11b34
+myst-parser>=0.13.7
+moto>=2.0.5
+sphinx-copybutton>=0.3.1

papermill/cli.py

@@ -201,6 +201,10 @@ def papermill(
     from stdin and write it out to stdout.
 
     """
+    # Jupyter deps use frozen modules, so we disable the python 3.11+ warning about debugger if running the CLI
+    if 'PYDEVD_DISABLE_FILE_VALIDATION' not in os.environ:
+        os.environ['PYDEVD_DISABLE_FILE_VALIDATION'] = '1'
+
     if not help_notebook:
         required_output_path = not (INPUT_PIPED or OUTPUT_PIPED)
         if required_output_path and not output_path:

papermill/tests/test_hdfs.py

@@ -1,6 +1,8 @@
 import unittest
 from unittest.mock import MagicMock, patch
 
+import pytest
+
 from ..iorw import HDFSHandler
 
 
@@ -38,6 +40,7 @@ def __init__(self, path):
         self.path = path
 
 
+@pytest.mark.skip(reason="No valid dep package for python 3.12 yet")
 @patch('papermill.iorw.HadoopFileSystem', side_effect=MockHadoopFileSystem())
 class HDFSTest(unittest.TestCase):
     def setUp(self):

requirements.txt

@@ -6,3 +6,4 @@ tqdm >= 4.32.2
 requests
 entrypoints
 tenacity >= 5.0.2
+aiohttp==3.9.0b0;python_version=="3.12"

setup.py

@@ -44,9 +44,12 @@ def read_reqs(fname, folder=None):
 gcs_reqs = read_reqs('gcs.txt', folder='requirements')
 hdfs_reqs = read_reqs('hdfs.txt', folder='requirements')
 github_reqs = read_reqs('github.txt', folder='requirements')
+docs_only_reqs = read_reqs('requirements.txt', folder='docs')
 black_reqs = ['black >= 19.3b0']
 all_reqs = s3_reqs + azure_reqs + gcs_reqs + hdfs_reqs + black_reqs
-dev_reqs = read_reqs('dev.txt', folder='requirements') + all_reqs
+docs_reqs = all_reqs + docs_only_reqs
+# Temporarily remove hdfs_reqs from dev deps until the pyarrow package is available for Python 3.12
+dev_reqs = read_reqs('dev.txt', folder='requirements') + s3_reqs + azure_reqs + gcs_reqs + black_reqs # all_reqs
 extras_require = {
     "test": dev_reqs,
     "dev": dev_reqs,
@@ -57,6 +60,7 @@ def read_reqs(fname, folder=None):
     "hdfs": hdfs_reqs,
     "github": github_reqs,
     "black": black_reqs,
+    "docs": docs_reqs,
 }
 
 # Get the long description from the README file
@@ -76,7 +80,7 @@ def read_reqs(fname, folder=None):
     long_description_content_type='text/markdown',
     url='https://github.com/nteract/papermill',
     packages=['papermill'],
-    python_requires='>=3.7',
+    python_requires='>=3.8',
     install_requires=read_reqs('requirements.txt'),
     extras_require=extras_require,
     entry_points={'console_scripts': ['papermill = papermill.__main__:papermill']},
@@ -93,9 +97,10 @@ def read_reqs(fname, folder=None):
         'License :: OSI Approved :: BSD License',
         'Programming Language :: Python',
         'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.7',
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
         'Programming Language :: Python :: 3.10',
+        'Programming Language :: Python :: 3.11',
+        'Programming Language :: Python :: 3.12',
     ],
 )

tox.ini

@@ -1,13 +1,14 @@
 [tox]
 skipsdist = true
-envlist = py{37,38,39,310}, flake8, dist, manifest, docs, binder
+envlist = py{38,39,310,311,312}, flake8, dist, manifest, docs, binder
 
 [gh-actions]
 python =
-    3.7: py37
-    3.8: py38, dist
+    3.8: py38
     3.9: py39
     3.10: py310
+    3.11: py311
+    3.12: py312, dist
 
 # Linters
 [testenv:flake8]
@@ -27,9 +28,7 @@ ignore =
 [testenv:docs]
 description = invoke sphinx-build to build the HTML docs
 deps =
-    .[dev]
-    .[azure]
-    -r docs/requirements.txt
+    .[docs]
 extras = docs
 commands =
     sphinx-build -d "{toxworkdir}/docs_doctree" docs "{toxworkdir}/docs_out" --color -W -bhtml {posargs}
@@ -45,16 +44,14 @@ commands = python -c "import glob; import papermill as pm; [pm.execute_notebook(
 # Distro
 [testenv:dist]
 skip_install = true
-# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line...
 commands =
     python setup.py sdist --dist-dir={distdir} bdist_wheel --dist-dir={distdir}
     /bin/bash -c 'python -m pip install -U --force-reinstall {distdir}/papermill*.whl'
-    /bin/bash -c 'python -m pip install -U --force-reinstall --no-deps {distdir}/papermill*.tar.gz'
 
 # Black
 [testenv:black]
 description = apply black linter with desired rules
-basepython = python3.8
+basepython = python3.12
 deps =
     black
 commands = black -S -l 100 .
@@ -67,15 +64,20 @@ setenv =
     AWS_SECRET_ACCESS_KEY=foobar_secret
 passenv = *
 basepython =
-    py37: python3.7
     py38: python3.8
     py39: python3.9
     py310: python3.10
-    flake8: python3.8
-    manifest: python3.8
-    dist: python3.8
-    docs: python3.8
-    binder: python3.8
-    black: python3.8
+    py311: python3.11
+    py312: python3.12
+    flake8: python3.12
+    manifest: python3.12
+    dist: python3.12
+    docs: python3.12
+    binder: python3.12
+    black: python3.12
 deps = .[dev]
+# Have to use /bin/bash or the `*` will cause that argument to get quoted by the tox command line...
+allowlist_externals = /bin/bash
+# Python 3.12 breaks default pip/setuptools versions ... force an upgrade of these before anything else
+install_command = /bin/bash ./tox_py_installer {opts} {packages}
 commands = pytest -v --maxfail=2 --cov=papermill -W always {posargs}

tox_py_installer

@@ -0,0 +1,5 @@
+#!/bin/bash
+python -m ensurepip --upgrade
+python -m pip install --upgrade setuptools
+# python -m pip install {opts} {packages}
+python -m pip install $1 $2