Merge branch 'main' into black-120

.github/dependabot.yml

@@ -9,10 +9,29 @@ updates:
     schedule:
       interval: "monthly"
     # Labels on pull requests for version updates only
-    labels:
-      - "ci"
+    labels: ["ci"]
     pull-request-branch-name:
       # Separate sections of the branch name with a hyphen for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1`
       separator: "-"
     # Allow up to 5 open pull requests for GitHub Actions
     open-pull-requests-limit: 5
+    reviewers:
+      - "borda"
+
+  # Enable version updates for python
+  - package-ecosystem: "pip"
+    # Look for a `requirements` in the `root` directory
+    directory: "/requirements"
+    # Check for updates once a week
+    schedule:
+      interval: "weekly"
+    # Labels on pull requests for version updates only
+    labels: ["enhancement"]
+    pull-request-branch-name:
+      # Separate sections of the branch name with a hyphen
+      # for example, `dependabot-npm_and_yarn-next_js-acorn-6.4.1`
+      separator: "-"
+    # Allow up to 5 open pull requests for pip dependencies
+    open-pull-requests-limit: 5
+    reviewers:
+      - "willingc"

.gitignore

@@ -123,3 +123,7 @@ pip-wheel-metadata
 
 # VIM swap files
 *.swp
+
+
+# lint
+.ruff_cache/

.pre-commit-config.yaml

@@ -25,7 +25,7 @@ repos:
     rev: v3.15.0
     hooks:
       - id: pyupgrade
-        args: [--py37-plus]
+        args: ["--py38-plus"]
         name: Upgrade code
 
   #- repo: https://github.com/myint/docformatter
@@ -34,10 +34,12 @@ repos:
   #    - id: docformatter
   #      args: [--in-place, --wrap-summaries=120, --wrap-descriptions=120]
 
-  #- repo: https://github.com/PyCQA/isort
-  #  rev: 5.10.1
-  #  hooks:
-  #    - id: isort
+  - repo: https://github.com/codespell-project/codespell
+    rev: v2.2.6
+    hooks:
+      - id: codespell
+        additional_dependencies: [tomli]
+        #args: ["--write-changes"] # uncomment if you want to get automatic fixing
 
   - repo: https://github.com/psf/black
     rev: 23.11.0
@@ -58,11 +60,8 @@ repos:
     hooks:
       - id: yesqa
 
-  - repo: https://github.com/pycqa/flake8
-    rev: 6.1.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.4
     hooks:
-      - id: flake8
-        arge:
-          - "--count"
-          - "--show-source"
-          - "--statistics"
+        - id: ruff
+          args: ["--fix"]

DEVELOPMENT_GUIDE.md

@@ -51,3 +51,28 @@ In `cli.py` you'll want to add an `@click.option` above the `papermill` method a
 Now in `execute.py`'s `execute_notebook` we want to add the appropriate argument and default it to something sane. Add the argument to the docstring as well. Then pass or use that argument where it's needed to achieve the desired effect. Usually these options get passed to `_execute_parameterized_notebook`.
 
 To update the tests you'll need both `test_cli.py` and `test_execute.py` to include the new option. Though the CLI tests need only check that the appropriate values get passed to `execute_notebook`.
+
+# Releasing
+
+## Prerequisites
+
+- First check that the CHANGELOG is up to date for the next release version
+- Ensure dev requirements are installed `pip install -r requirements/dev.txt`
+
+## Push to GitHub
+
+Change from patch to minor or major for appropriate version updates.
+
+```bash
+bumpversion patch
+git push upstream && git push upstream --tags
+```
+
+## Push to PyPI
+
+```bash
+rm -rf dist/*
+rm -rf build/*
+python setup.py sdist bdist_wheel
+twine upload dist/*
+```

docs/changelog.md

@@ -12,19 +12,19 @@
 - Added “github” extra deps. to the “all” extra [PR #715](https://github.com/nteract/papermill/pull/715)
 - Github actions updated to latest version [PR #732](https://github.com/nteract/papermill/pull/732)
 - Added a warning if non set parameter is passed through CLI [PR #701](https://github.com/nteract/papermill/pull/701)
-- Skip black formatting when encounting attribute errors [PR #699](https://github.com/nteract/papermill/pull/699)
-- Removed fored deepcopy of notebook objects [PR #694](https://github.com/nteract/papermill/pull/694)
+- Skip black formatting when encountering attribute errors [PR #699](https://github.com/nteract/papermill/pull/699)
+- Removed forced deepcopy of notebook objects [PR #694](https://github.com/nteract/papermill/pull/694)
 - Always update notebook version on execute [PR #691](https://github.com/nteract/papermill/pull/691)
 - Set minimum version for tenacity [PR #682](https://github.com/nteract/papermill/pull/682)
 - Removed use of ansiwrap [PR #681](https://github.com/nteract/papermill/pull/681)
-- Added precommit hook [PR #678](https://github.com/nteract/papermill/pull/678)
+- Added pre-commit hook [PR #678](https://github.com/nteract/papermill/pull/678)
 - Added dependabot for GHA [PR #677](https://github.com/nteract/papermill/pull/677)
 
 ## 2.4.0
 
 - Add tracking cell executions with cell descriptions [PR #650](https://github.com/nteract/papermill/pull/650)
 - Fixed Azure Blob URI matching for prefixes [PR #654](https://github.com/nteract/papermill/pull/654)
-- Updates HDFS handler to use PyArrow.fs.HadoopFileSystm [PR #658](https://github.com/nteract/papermill/pull/658)
+- Updates HDFS handler to use `PyArrow.fs.HadoopFileSystm` [PR #658](https://github.com/nteract/papermill/pull/658)
 - Drop support for Python 3.6 [PR #666](https://github.com/nteract/papermill/pull/666)
 - CI Build fixes [PR #664](https://github.com/nteract/papermill/pull/664)
 - Updated mock references in tests [PR #668](https://github.com/nteract/papermill/pull/668)
@@ -224,7 +224,7 @@ pip install papermill[all]
 
 - Optional IO extensions are now separated into different dependencies.
 - Added gs:// optional dependency for google cloud storage support.
-- null json fields in parmaeters now translate correctly to equivilent fields in each supported language
+- null json fields in parameters now translate correctly to equivalent fields in each supported language
 
 ### Fixes
 
@@ -240,7 +240,7 @@ pip install papermill[all]
 ### Features
 
 - Log level can now be set with `--log-level`
-- The working directory of papermill can be set with the `--cwd` option. This will set the executing context of the kernel but not impact input/output paths. `papermill --cwd foo bar/input_nb.ipynb bar/output_nb.ipynb` would make the notebook able to reference files in the `foo` directoy without `../foo` but still save the output notebook in the `bar` directory.
+- The working directory of papermill can be set with the `--cwd` option. This will set the executing context of the kernel but not impact input/output paths. `papermill --cwd foo bar/input_nb.ipynb bar/output_nb.ipynb` would make the notebook able to reference files in the `foo` directory without `../foo` but still save the output notebook in the `bar` directory.
 - Tox has been added for testing papermill. This makes it easier to catch linting and manifest issues without waiting for a failed Travis build.
 
 ### Fixes
@@ -274,7 +274,7 @@ pip install papermill[all]
 - Added engines abstraction and command line argument
 - Moved some nbconvert wrappers out of papermill
 - Added Azure blob storage support
-- Fixed botocore upgrade comptability issue (all version of boto now supported again)
+- Fixed botocore upgrade compatibility issue (all version of boto now supported again)
 - Removed whitelisted environment variable assignment
 
 ## 0.15.1
@@ -286,7 +286,7 @@ pip install papermill[all]
 - Warnings added when an unexpected file extension is used
 - Papermill version is visible to the CLI
 - More messages us logging module now (and can be filtered accordingly)
-- Binder link from README was greatly improved to demostrate papermill features
+- Binder link from README was greatly improved to demonstrate papermill features
 
 ## 0.15.0
 
@@ -333,7 +333,7 @@ pip install papermill[all]
 ## 0.12.6
 
 - Changed CLI outputs from papermill messaging to stderr
-- Changed IOResolvers to perseve ordering of definition in resolving paths
+- Changed IOResolvers to persevere ordering of definition in resolving paths
 
 ## 0.12.5
 

papermill/__init__.py

@@ -1,5 +1,4 @@
-from .version import version as __version__
-
-from .exceptions import PapermillException, PapermillExecutionError
-from .execute import execute_notebook
-from .inspection import inspect_notebook
+from .exceptions import PapermillException, PapermillExecutionError  # noqa: F401
+from .execute import execute_notebook  # noqa: F401
+from .inspection import inspect_notebook  # noqa: F401
+from .version import version as __version__  # noqa: F401

papermill/abs.py

@@ -1,9 +1,9 @@
 """Utilities for working with Azure blob storage"""
-import re
 import io
+import re
 
-from azure.storage.blob import BlobServiceClient
 from azure.identity import EnvironmentCredential
+from azure.storage.blob import BlobServiceClient
 
 
 class AzureBlobStore:
@@ -29,7 +29,7 @@ def _blob_service_client(self, account_name, sas_token=None):
     @classmethod
     def _split_url(self, url):
         """
-        see: https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1  # noqa: E501
+        see: https://docs.microsoft.com/en-us/azure/storage/common/storage-dotnet-shared-access-signature-part-1
         abs://myaccount.blob.core.windows.net/sascontainer/sasblob.txt?sastoken
         """
         match = re.match(r"abs://(.*)\.blob\.core\.windows\.net\/(.*?)\/([^\?]*)\??(.*)$", url)

papermill/cli.py

@@ -1,23 +1,21 @@
 """Main `papermill` interface."""
 
+import base64
+import logging
 import os
+import platform
 import sys
-from stat import S_ISFIFO
-import nbclient
 import traceback
-
-import base64
-import logging
+from stat import S_ISFIFO
 
 import click
-
+import nbclient
 import yaml
-import platform
 
 from .execute import execute_notebook
-from .iorw import read_yaml_file, NoDatesSafeLoader
 from .inspection import display_notebook_help
-from . import __version__ as papermill_version
+from .iorw import NoDatesSafeLoader, read_yaml_file
+from .version import version as papermill_version
 
 click.disable_unicode_literals_warning = True
 

papermill/clientwrap.py

@@ -1,5 +1,5 @@
-import sys
 import asyncio
+import sys
 
 from nbclient import NotebookClient
 from nbclient.exceptions import CellExecutionError

papermill/engines.py

@@ -1,16 +1,16 @@
 """Engines to perform different roles"""
-import sys
 import datetime
-import dateutil
-
+import sys
 from functools import wraps
+
+import dateutil
 import entrypoints
 
-from .log import logger
-from .exceptions import PapermillException
 from .clientwrap import PapermillNotebookClient
+from .exceptions import PapermillException
 from .iorw import write_ipynb
-from .utils import merge_kwargs, remove_args, nb_kernel_name, nb_language
+from .log import logger
+from .utils import merge_kwargs, nb_kernel_name, nb_language, remove_args
 
 
 class PapermillEngines:
@@ -106,7 +106,7 @@ def __init__(self, nb, output_path=None, log_output=False, progress_bar=True, au
         self.last_save_time = self.now()  # Not exactly true, but simplifies testing logic
         self.pbar = None
         if progress_bar:
-            # lazy import due to implict slow ipython import
+            # lazy import due to implicit slow ipython import
             from tqdm.auto import tqdm
 
             self.pbar = tqdm(total=len(self.nb.cells), unit="cell", desc="Executing")

papermill/exceptions.py

@@ -30,7 +30,7 @@ def __init__(self, cell_index, exec_count, source, ename, evalue, traceback):
 
     def __str__(self):
         # Standard Behavior of an exception is to produce a string representation of its arguments
-        # when called with str(). In order to maintain compatability with previous versions which
+        # when called with str(). In order to maintain compatibility with previous versions which
         # passed only the message to the superclass constructor, __str__ method is implemented to
         # provide the same result as was produced in the past.
         message = "\n" + 75 * "-" + "\n"

papermill/execute.py

@@ -1,13 +1,14 @@
-import nbformat
 from pathlib import Path
 
-from .log import logger
-from .exceptions import PapermillExecutionError
-from .iorw import get_pretty_path, local_file_io_cwd, load_notebook_node, write_ipynb
+import nbformat
+
 from .engines import papermill_engines
-from .utils import chdir
-from .parameterize import add_builtin_parameters, parameterize_notebook, parameterize_path
+from .exceptions import PapermillExecutionError
 from .inspection import _infer_parameters
+from .iorw import get_pretty_path, load_notebook_node, local_file_io_cwd, write_ipynb
+from .log import logger
+from .parameterize import add_builtin_parameters, parameterize_notebook, parameterize_path
+from .utils import chdir
 
 
 def execute_notebook(

papermill/inspection.py

@@ -1,7 +1,8 @@
 """Deduce parameters of a notebook from the parameters cell."""
-import click
 from pathlib import Path
 
+import click
+
 from .iorw import get_pretty_path, load_notebook_node, local_file_io_cwd
 from .log import logger
 from .parameterize import add_builtin_parameters, parameterize_path

papermill/iorw.py

@@ -1,18 +1,16 @@
+import fnmatch
+import json
 import os
 import sys
-import json
-import yaml
-import fnmatch
-import nbformat
-import requests
 import warnings
-import entrypoints
-
 from contextlib import contextmanager
 
+import entrypoints
+import nbformat
+import requests
+import yaml
 from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential
 
-from . import __version__
 from .exceptions import (
     PapermillException,
     PapermillRateLimitException,
@@ -21,6 +19,7 @@
 )
 from .log import logger
 from .utils import chdir
+from .version import version as __version__
 
 try:
     from .s3 import S3
@@ -48,7 +47,7 @@
     GCSFileSystem = missing_dependency_generator("gcsfs", "gcs")
 
 try:
-    from pyarrow.fs import HadoopFileSystem, FileSelector
+    from pyarrow.fs import FileSelector, HadoopFileSystem
 except ImportError:
     HadoopFileSystem = missing_dependency_generator("pyarrow", "hdfs")