Add more CI

.github/scripts/typeshed_third_party.py

@@ -0,0 +1,65 @@
+"""Script that attempts to invoke docstring-adder on all third-party typeshed stubs."""
+
+import argparse
+import subprocess
+import tempfile
+from pathlib import Path
+
+
+def add_docstrings(typeshed_dir: Path, docstring_adder_dir: Path) -> None:
+    """Add docstrings to third-party typeshed stubs."""
+
+    for path in (typeshed_dir / "stubs").iterdir():
+        with tempfile.TemporaryDirectory() as td:
+            venv_dir = f"{td}-venv"
+            subprocess.run(["uv", "venv", "--python", "3.13", venv_dir], check=True)
+
+            # seems to fail with an internal assertion when you try to import it...?
+            if path.name == "keyboard":
+                continue
+
+            try:
+                subprocess.run(
+                    [
+                        "uvx",
+                        "--python",
+                        f"{venv_dir}/bin/python",
+                        "--from",
+                        docstring_adder_dir,
+                        "add-docstrings",
+                        "--typeshed-packages",
+                        path,
+                    ],
+                    check=True,
+                )
+            except subprocess.CalledProcessError as e:
+                if e.returncode == 10:
+                    print(f"\nFailed to install runtime package for `{path.name}\n`")
+                    continue
+                else:
+                    raise
+
+
+if __name__ == "__main__":
+
+    def absolute_path(path: str) -> Path:
+        """Convert a string path to an absolute Path."""
+        return Path(path).absolute()
+
+    parser = argparse.ArgumentParser(__doc__)
+    parser.add_argument(
+        "--typeshed-dir",
+        type=absolute_path,
+        help="Path to the typeshed directory.",
+        required=True,
+    )
+    parser.add_argument(
+        "--docstring-adder-dir",
+        type=absolute_path,
+        help="Path to docstring-adder.",
+        required=True,
+    )
+    args = parser.parse_args()
+    add_docstrings(
+        typeshed_dir=args.typeshed_dir, docstring_adder_dir=args.docstring_adder_dir
+    )

.github/workflows/check.yaml

@@ -6,6 +6,8 @@ on:
       - main
   pull_request:
   workflow_dispatch:
+  schedule:
+    - cron: "0 12 * * *"
 
 permissions:
   contents: read
@@ -14,14 +16,15 @@ env:
   FORCE_COLOR: 1 # Request colored output from CLI tools supporting it
   CLICOLOR_FORCE: 1 # recognized by uv
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  DEFAULT_PYTHON_VERSION: "3.13" # Default Python version for uvx
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 jobs:
-  check:
-    name: Check
+  stdlib:
+    name: Check typeshed stdlib
     runs-on: ${{ matrix.platform }}
     strategy:
       fail-fast: false
@@ -39,7 +42,7 @@ jobs:
       - name: Check a basic run doesn't fail
         run: uvx --python=${{ matrix.python-version }} --force-reinstall --from=./docstring-adder add-docstrings --stdlib-path ./typeshed/stdlib
       - name: Check typeshed's mypy_test.py passes
-        run: uv run --directory=typeshed --no-project --python=3.13 --with-requirements=requirements-tests.txt python tests/mypy_test.py stdlib -p "${{ matrix.python-version }}"
+        run: uv run --directory=typeshed --no-project --python="${DEFAULT_PYTHON_VERSION}" --with-requirements=requirements-tests.txt python tests/mypy_test.py stdlib -p "${{ matrix.python-version }}"
       - name: Check running it again doesn't produce any changes
         # Some weird thing with distutils means that it does actually add docstrings
         # when run the second time that it didn't add when run the first time.
@@ -51,10 +54,57 @@ jobs:
           cd typeshed
           git config --global user.email "you@example.com"
           git config --global user.name "Your Name"
+          test -n "$(git status --porcelain)"
           git commit -am "."
           uvx --python=${{ matrix.python-version }} --force-reinstall --from=../docstring-adder add-docstrings --stdlib-path ./stdlib
           test -z "$(git status --porcelain)"
 
+  typeshed-third-party:
+    name: Check typeshed third-party stubs
+    runs-on: ubuntu-latest
+    # Too slow to run on every PR
+    if: ${{ github.event_name != 'pull_request' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: docstring-adder
+          persist-credentials: false
+      - name: Checkout typeshed
+        run: git clone --depth=1 https://github.com/python/typeshed.git || git clone --depth=1 https://github.com/python/typeshed.git
+      - uses: astral-sh/setup-uv@v6
+      - name: Check a basic run doesn't fail
+        run: uvx --python="${DEFAULT_PYTHON_VERSION}" python ./docstring-adder/.github/scripts/typeshed_third_party.py --typeshed-dir ./typeshed --docstring-adder-dir ./docstring-adder
+      - name: Check typeshed's mypy_test.py passes
+        run: uv run --directory=typeshed --no-project --python="${DEFAULT_PYTHON_VERSION}" --with-requirements=requirements-tests.txt python tests/mypy_test.py stubs -p "${DEFAULT_PYTHON_VERSION}"
+      # Ideally we'd now check that running it again doesn't produce any changes, but it fails for some third-party packages.
+      # I believe this is because some badly-behaved packages make persistent changes to the Python environment when they're imported.
+      # Stubs packages for which this fails are: `Authlib`, `aws-xray-sdk`, `beautifulsoup4`, `html5lib`, `python-jose`, `qrcode`
+
+  # A fairly arbitrary check, but it's good to enforce that it works on a non-typeshed stubs package.
+  pandas-stubs:
+    name: Check running on pandas-stubs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          path: docstring-adder
+          persist-credentials: false
+      - name: Checkout typeshed
+        run: git clone --depth=1 https://github.com/pandas-dev/pandas-stubs.git || git clone --depth=1 https://github.com/pandas-dev/pandas-stubs.git
+      - uses: astral-sh/setup-uv@v6
+      - name: Check a basic run doesn't fail
+        run: uvx --python="${DEFAULT_PYTHON_VERSION}" --force-reinstall --with=pandas --from=./docstring-adder add-docstrings --packages ./pandas-stubs
+      - name: Check running it again doesn't produce any changes
+        shell: bash
+        run: |
+          git config --global user.email "you@example.com"
+          git config --global user.name "Your Name"
+          cd pandas-stubs
+          test -n "$(git status --porcelain)"
+          git commit -am "."
+          uvx --python="${DEFAULT_PYTHON_VERSION}" --force-reinstall --with=pandas --from=../docstring-adder add-docstrings --packages .
+          test -z "$(git status --porcelain)"
+
   mypy:
     name: Run mypy
     runs-on: ubuntu-latest
@@ -73,3 +123,28 @@ jobs:
           persist-credentials: false
       - uses: actions/setup-python@v5
       - uses: pre-commit/action@v3.0.1
+
+  create-issue-on-failure:
+    name: Create an issue if the daily test run failed
+    runs-on: ubuntu-latest
+    needs:
+      - stdlib
+      - typeshed-third-party
+      - mypy
+      - pre-commit
+      - pandas-stubs
+    if: ${{ github.repository == 'astral-sh/docstring-adder' && always() && github.event_name == 'schedule' && ((needs.stdlib.result == 'failure') || (needs.typeshed-third-party.result == 'failure') || (needs.mypy.result == 'failure') || (needs.pre-commit.result == 'failure') || (needs.pandas-stubs.result == 'failure')) }}
+    permissions:
+      issues: write
+    steps:
+      - uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            await github.rest.issues.create({
+              owner: "astral-sh",
+              repo: "docstring-adder",
+              title: `Daily test run failed on ${new Date().toDateString()}`,
+              body: "Run listed here: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+              labels: ["bug"],
+            })

add_docstrings.py

@@ -32,6 +32,8 @@
 Some miscellaneous details:
 - The tool should be idempotent. It should add docstrings where possible, but it should
   never remove or alter docstrings that were already present in the stub file.
+  Idempotency cannot be guaranteed, however, if importing the package at runtime causes
+  persistent changes to be made to your Python environment.
 - Because it is a libcst-based codemod, it should not make spurious changes to formatting
   or comments in the stub file. `type: ignore` comments should be preserved; mypy and
   other type checkers should still be able to type-check the stub file after
@@ -910,7 +912,7 @@ def _main() -> None:
                 add_docstrings_to_stub(module, context, stdlib_blacklist)
         elif any(path.is_relative_to(p) for p in package_paths):
             add_docstrings_to_stub(module, context, combined_blacklist)
-    m = "\n--- Successfully codemodded typeshed ---"
+    m = "\n--- Successfully completed the codemod ---"
     print(colored(m, "green"))
     sys.exit(0)