Run stubtest only for newly released third-party package version

.github/workflows/daily.yml

@@ -53,9 +53,10 @@ jobs:
       - name: Run stubtest
         run: python tests/stubtest_stdlib.py
 
+  # This job only runs when requirements.txt is changed
   stubtest-third-party:
     name: "stubtest: third party"
-    if: ${{ github.repository == 'python/typeshed' || github.event_name != 'schedule' }}
+    if: ${{ github.repository == 'python/typeshed' && github.event_name != 'schedule' }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -136,8 +137,8 @@ jobs:
   create-issue-on-failure:
     name: Create issue on failure
     runs-on: ubuntu-latest
-    needs: [stubtest-stdlib, stubtest-third-party, stub-uploader]
-    if: ${{ github.repository == 'python/typeshed' && always() && github.event_name == 'schedule' && (needs.stubtest-stdlib.result == 'failure' || needs.stubtest-third-party.result == 'failure' || needs.stub-uploader.result == 'failure') }}
+    needs: [stubtest-stdlib, stub-uploader]
+    if: ${{ github.repository == 'python/typeshed' && always() && github.event_name == 'schedule' && (needs.stubtest-stdlib.result == 'failure' || needs.stub-uploader.result == 'failure') }}
     permissions:
       issues: write
     steps:

.github/workflows/stubsabot.yml

@@ -18,6 +18,8 @@ jobs:
     name: Upgrade stubs with stubsabot
     if: github.repository == 'python/typeshed'
     runs-on: ubuntu-latest
+    outputs:
+      STUBS: ${{ steps.runstubsabot.outputs.STUBS }}
     steps:
       - uses: actions/checkout@v5
         with:
@@ -37,14 +39,97 @@ jobs:
       - name: Install dependencies
         run: uv pip install -r requirements-tests.txt --system
       - name: Run stubsabot
-        run: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python scripts/stubsabot.py --action-level everything
+        id: runstubsabot
+        shell: bash
+        run: |
+          # Parse stubsabot.py output to find stubs that should be tested.
+          exec 5>&1
+          EXIT_FILE=$(mktemp)
+          STUBS_FILE=$(mktemp)
+          script -q /dev/null -c '
+            GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} python scripts/stubsabot.py --action-level everything
+            echo $? > '"$EXIT_FILE"'
+          ' 2>&1 | while IFS= read -r line; do
+              echo "$line" >&5
+              clean=$(echo "$line" | sed -r "s/\x1B\[[0-9;]*[mK]//g")
+              if [[ $clean == *"should be tested by stubtest"* ]]; then
+                  echo "${clean%% *}" >> "$STUBS_FILE"
+              fi
+          done
+          exit_code=$(cat "$EXIT_FILE")
+          STUBS=$(xargs < "$STUBS_FILE")
+          echo "STUBS=$STUBS" >> $GITHUB_OUTPUT
+          exit $exit_code
+
+  stubtest-third-party:
+    name: "stubtest: third party"
+    if: github.repository == 'python/typeshed'
+    runs-on: ${{ matrix.os }}
+    needs: [stubsabot]
+    strategy:
+      matrix:
+        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
+      fail-fast: false
+    env:
+      STUBS: ${{ needs.stubsabot.outputs.STUBS }}
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"
+          cache: pip
+          cache-dependency-path: |
+            requirements-tests.txt
+            stubs/**/METADATA.toml
+      - name: Install dependencies
+        run: pip install -r requirements-tests.txt
+      - name: Install required system packages
+        shell: bash
+        run: |
+          if [ -n "$STUBS" ]; then
+            PACKAGES=$(python tests/get_stubtest_system_requirements.py $STUBS)
+            if [ "${{ runner.os }}" = "Linux" ]; then
+              if [ -n "$PACKAGES" ]; then
+                printf "Installing APT packages:\n  $(echo $PACKAGES | sed 's/ /\n  /g')\n"
+                sudo apt-get update -q && sudo apt-get install -qy $PACKAGES
+              fi
+            else
+              if [ "${{ runner.os }}" = "macOS" ] && [ -n "$PACKAGES" ]; then
+                printf "Installing Homebrew packages:\n  $(echo $PACKAGES | sed 's/ /\n  /g')\n"
+                brew install -q $PACKAGES
+              fi
+
+              if [ "${{ runner.os }}" = "Windows" ] && [ -n "$PACKAGES" ]; then
+                printf "Installing Chocolatey packages:\n  $(echo $PACKAGES | sed 's/ /\n  /g')\n"
+                choco install -y $PACKAGES
+              fi
+            fi
+          fi
+      - name: Run stubtest
+        shell: bash
+        run: |
+          if [ -n "$STUBS" ]; then
+            echo "Testing $STUBS..."
+
+            if [ "${{ runner.os }}" = "Linux" ]; then
+              PYTHON_EXECUTABLE="xvfb-run python"
+            else
+              PYTHON_EXECUTABLE="python"
+            fi
+
+            $PYTHON_EXECUTABLE tests/stubtest_third_party.py --ci-platforms-only $STUBS
+          else
+            echo "Nothing to test"
+          fi
 
   # https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2
   create-issue-on-failure:
     name: Create issue on failure
     runs-on: ubuntu-latest
-    needs: [stubsabot]
-    if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubsabot.result == 'failure') }}
+    needs: [stubsabot, stubtest-third-party]
+    if: ${{ github.repository == 'python/typeshed' && always() && (needs.stubsabot.result == 'failure' || needs.stubtest-third-party.result == 'failure') }}
     steps:
       - uses: actions/github-script@v8
         with:

scripts/stubsabot.py

@@ -620,6 +620,15 @@ async def has_no_longer_updated_release(release_to_download: PypiReleaseDownload
     return await with_extracted_archive(release_to_download, session=session, handler=parse_no_longer_updated_from_archive)
 
 
+def is_new_release(upload_datetime: datetime.datetime) -> bool:
+    if upload_datetime.tzinfo is None:
+        upload_datetime = upload_datetime.replace(tzinfo=datetime.timezone.utc)
+    else:
+        upload_datetime = upload_datetime.astimezone(datetime.timezone.utc)
+    yesterday = datetime.datetime.now(tz=datetime.timezone.utc) - datetime.timedelta(days=1)
+    return upload_datetime > yesterday
+
+
 async def determine_action(distribution: str, session: aiohttp.ClientSession) -> Update | NoUpdate | Obsolete | Remove | Error:
     try:
         return await determine_action_no_error_handling(distribution, session)
@@ -663,6 +672,9 @@ async def determine_action_no_error_handling(
     latest_version = latest_release.version
     obsolete_since = await find_first_release_with_py_typed(pypi_info, session=session)
     if obsolete_since is None and latest_version in stub_info.version_spec:
+        if is_new_release(latest_release.upload_date):
+            # Next print should be parsed by github action, see `stubsabot.yml`
+            print(colored(f"{stub_info.distribution} should be tested by stubtest", "blue"))
         return NoUpdate(stub_info.distribution, "up to date")
 
     relevant_version = obsolete_since.version if obsolete_since else latest_version