Shard release publishing and signing across a gated matrix (aio-libs#13226)

Author: bdraco

.github/workflows/ci-cd.yml

@@ -753,7 +753,7 @@ jobs:
         path: ./wheelhouse/*.whl
 
   deploy:
-    name: Deploy
+    name: Deploy (${{ matrix.group }})
     needs:
     - build-tarball
     - build-wheels
@@ -766,12 +766,54 @@ jobs:
       contents: write  # IMPORTANT: mandatory for making GitHub Releases
       id-token: write  # IMPORTANT: mandatory for trusted publishing & sigstore
 
+    # TAG is shared by the two release-existence steps. GITHUB_TOKEN stays scoped
+    # to the steps that need it rather than job-wide, so third-party actions in
+    # this job never see it in their environment.
+    env:
+      TAG: ${{ github.ref_name }}
+
+    # The required-reviewer pypi environment gates this job, so a human must
+    # approve before anything is created or published. Release creation and
+    # publishing all live in this one gated matrix, so a release needs a single
+    # approval: the groups are pending together and a reviewer approves them in
+    # one review (see the strategy comment below).
     environment:
       name: pypi
       url: https://pypi.org/p/aiohttp
 
+    strategy:
+      # The PyPI publish and the Sigstore signing each mint one short-lived OIDC
+      # identity per job and reuse it for every file, so signing the whole dist
+      # set in a single job can outlast the token and fail partway through
+      # (pypa/gh-action-pypi-publish#307). Splitting the work across groups, each
+      # its own job with a fresh identity signing only its share, keeps every
+      # signing loop well under the token lifetime.
+      #
+      # The groups run in parallel and all target the pypi environment, so they
+      # are pending for approval at the same time and a reviewer approves them in
+      # a single review rather than one prompt per group. The first group
+      # (job-index 0) creates the GitHub Release and the others wait for it; each
+      # group only ever touches its own disjoint share of dists, so the
+      # concurrent Release asset uploads never collide. fail-fast is off and
+      # every step is idempotent, so a single failed group can be re-run on its
+      # own.
+      #
+      # Each label "N of M" self-encodes its own position and total, which is the
+      # only source of truth for the split. Keep `group` the sole matrix axis: an
+      # include/exclude entry would renumber strategy.job-index / job-total, but
+      # the label-derived split below stays correct as long as job-index 0 is the
+      # first label.
+      fail-fast: false
+      matrix:
+        group:
+        - 1 of 2
+        - 2 of 2
+
     steps:
     - name: Checkout
+      # Only the release-creating group needs the repo (create-release reads
+      # CHANGES.rst and aiohttp/__init__.py); the others only touch dist/.
+      if: ${{ strategy.job-index == 0 }}
       uses: actions/checkout@v7
       with:
         submodules: true
@@ -784,32 +826,64 @@ jobs:
         path: dist
         pattern: dist-*
         merge-multiple: true
-    - name: Collected dists
+    - name: Select this group's distributions
+      # Keep only this group's share of the dists so the job signs a bounded set.
+      # index and count come from the "N of M" label, the single source of truth
+      # for the split; to add a group, extend the matrix list above (e.g.
+      # "1 of 3" .. "3 of 3").
+      #
+      # The split is fully deterministic: the same built dists always sort the
+      # same way (LC_ALL=C, byte order, independent of runner locale) and land in
+      # the same group, so re-running a single failed group reprocesses exactly
+      # its own share and never touches another group's dists.
+      id: group
+      shell: bash
+      env:
+        GROUP: ${{ matrix.group }}
       run: |
-        tree dist
+        set -euo pipefail
+        index=$(( ${GROUP%% of *} - 1 ))
+        count=${GROUP##* of }
+        shopt -s nullglob
+        mapfile -t all < <(printf '%s\n' dist/*.whl dist/*.tar.gz | LC_ALL=C sort)
+        i=0
+        inputs=()
+        for f in "${all[@]}"; do
+          if [ "$(( i % count ))" -eq "${index}" ]; then
+            inputs+=("${f}")
+          else
+            rm -f -- "${f}"
+          fi
+          i=$(( i + 1 ))
+        done
+        echo "Group ${GROUP} keeps ${#inputs[@]} of ${#all[@]} dist(s):"
+        printf '  %s\n' "${inputs[@]}"
+        echo "sigstore-inputs=${inputs[*]}" >> "${GITHUB_OUTPUT}"
     - name: Check whether the GitHub Release already exists
-      # Allows re-running the deploy job after a partial failure (e.g. PyPI
-      # upload error) without the Make Release step failing with HTTP 422
-      # because the tag/release was created on a prior attempt. Treat
-      # only the literal `release not found` reply as "does not exist";
-      # other failures (auth, rate-limit, network) re-raise so the job
-      # fails loudly instead of falling through to Make Release.
+      # The first group owns Release creation. Skipping Make Release when the
+      # release already exists lets the job be re-run after a partial failure
+      # without hitting HTTP 422. Query the API and branch on the HTTP status,
+      # not on prose: a 404 means "create it", any other failure (auth,
+      # rate-limit, network) re-raises so the job fails loudly.
+      if: ${{ strategy.job-index == 0 }}
       id: gh-release
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        TAG: ${{ github.ref_name }}
       run: |
-        if gh release view "${TAG}" --repo "${GITHUB_REPOSITORY}" \
-            >/dev/null 2>err; then
+        set -euo pipefail
+        if gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \
+            --silent 2>err; then
           echo 'exists=true' >> "${GITHUB_OUTPUT}"
-        elif grep -qx 'release not found' err; then
+        elif grep -q 'HTTP 404' err; then
           echo 'exists=false' >> "${GITHUB_OUTPUT}"
         else
           cat err >&2
           exit 1
         fi
     - name: Make Release
-      if: steps.gh-release.outputs.exists != 'true'
+      # The first group creates the Release and uploads its share of the
+      # packages; the other groups add their packages and signatures below.
+      if: ${{ strategy.job-index == 0 && steps.gh-release.outputs.exists != 'true' }}
       uses: aio-libs/create-release@v1.6.6
       with:
         changes_file: CHANGES.rst
@@ -821,28 +895,52 @@ jobs:
           :issue:`(\d+)`
         fix_issue_repl: >-
           #\1
-
-    - name: >-
-        Publish 🐍📦 to PyPI
+    - name: Wait for the GitHub Release
+      # The other groups do not create the Release; they wait for the first
+      # group to create it before they publish or upload anything, so a failure
+      # to create the Release blocks the irreversible PyPI upload too. Only a
+      # 404 counts as "not yet"; any other API failure re-raises immediately
+      # instead of silently retrying for the whole timeout.
+      if: ${{ strategy.job-index != 0 }}
+      env:
+        GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        set -euo pipefail
+        for _ in $(seq 1 150); do
+          if gh api "repos/${GITHUB_REPOSITORY}/releases/tags/${TAG}" \
+              --silent 2>err; then
+            exit 0
+          fi
+          if ! grep -q 'HTTP 404' err; then
+            cat err >&2
+            exit 1
+          fi
+          sleep 2
+        done
+        echo "GitHub Release ${TAG} did not appear in time" >&2
+        exit 1
+    - name: Publish 🐍📦 to PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
-        # Allow re-running the deploy job after a partial PyPI upload
-        # without failing on dists that were already published.
+        # Allow re-running after a partial PyPI upload without failing on
+        # dists that a prior attempt already published.
         skip-existing: true
-
     - name: Sign the dists with Sigstore
       uses: sigstore/gh-action-sigstore-python@v3.4.0
       with:
-        inputs: >-
-          ./dist/*.tar.gz
-          ./dist/*.whl
-
+        inputs: ${{ steps.group.outputs.sigstore-inputs }}
     - name: Upload artifact signatures to GitHub Release
       # Confusingly, this action also supports updating releases, not
-      # just creating them. This is what we want here, since we've manually
+      # just creating them. This is what we want here, since the first group
       # created the release above.
+      #
+      # The groups run this concurrently against the same release, which is safe:
+      # each group's files are a disjoint share, so asset names never collide, and
+      # with no body/name inputs the action preserves the existing release
+      # metadata (it writes back what it reads) rather than clearing it, so the
+      # concurrent metadata updates are identical no-ops. The Wait step above
+      # guarantees the release (with its notes) already exists first.
       uses: softprops/action-gh-release@v3.0.2
       with:
-        # dist/ contains the built packages, which smoketest-artifacts/
-        # contains the signatures and certificates.
+        # dist/ holds this group's packages plus their Sigstore signatures.
         files: dist/**

CHANGES/13226.contrib.rst

@@ -0,0 +1 @@
+Split release publishing and signing across multiple jobs so each stays within the short-lived signing token lifetime, fixing intermittent release upload failures -- by :user:`bdraco`.