From a571f1e128eabff306fc741c1250b824d8931a41 Mon Sep 17 00:00:00 2001 From: mosfet80 Date: Mon, 23 Sep 2024 11:52:50 +0200 Subject: [PATCH 01/44] Update pylint lib https://github.com/pylint-dev/pylint/releases/tag/v3.3.0 --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 090a0d95..bcdbd2a1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -115,7 +115,7 @@ repos: language_version: python3.11 # flake8-commas doesn't work w/ Python 3.12 - repo: https://github.com/PyCQA/pylint.git - rev: v3.1.0 + rev: v3.3.0 hooks: - id: pylint args: From 00b87c80e8474c1b36b186257e09785ce3a5c288 Mon Sep 17 00:00:00 2001 From: mosfet80 Date: Mon, 23 Sep 2024 11:56:13 +0200 Subject: [PATCH 02/44] Update check-jsonschema and pre-commit libs https://github.com/python-jsonschema/check-jsonschema/releases https://github.com/pre-commit/pre-commit-hooks/releases/tag/v4.6.0 --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 090a0d95..c4590da7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,7 +22,7 @@ repos: - id: remove-tabs - repo: https://github.com/python-jsonschema/check-jsonschema.git - rev: 0.28.1 + rev: 0.29.2 hooks: - id: check-github-actions - id: check-github-workflows @@ -37,7 +37,7 @@ repos: - id: check-readthedocs - repo: https://github.com/pre-commit/pre-commit-hooks.git - rev: v4.5.0 + rev: v4.6.0 hooks: # Side-effects: - id: end-of-file-fixer From a1ce3844ac33bd8deec3df588c16ea681915ab7e Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Fri, 27 Sep 2024 20:47:02 +0200 Subject: [PATCH 03/44] Check for Trusted Publishing in magic link logic --- twine-upload.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/twine-upload.sh b/twine-upload.sh index fce45172..98d41b1b 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -73,7 +73,11 @@ MAGIC_LINK_MESSAGE="::warning title=Create a Trusted Publisher::\ A new Trusted Publisher for the currently running publishing workflow can be created \ by accessing the following link(s) while logged-in as an owner of the package(s):" -if [[ ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org || ${#PACKAGE_NAMES[@]} -eq 0 ]] ; then + +[[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ + && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false + +if [[ "${TRUSTED_PUBLISHING}" == true || ! "${INPUT_REPOSITORY_URL}" =~ pypi\.org || ${#PACKAGE_NAMES[@]} -eq 0 ]] ; then TRUSTED_PUBLISHING_MAGIC_LINK_NUDGE="" else if [[ "${INPUT_REPOSITORY_URL}" =~ test\.pypi\.org ]] ; then @@ -90,8 +94,6 @@ else echo "${MAGIC_LINK_MESSAGE}" >> $GITHUB_STEP_SUMMARY fi -[[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ - && TRUSTED_PUBLISHING=true || TRUSTED_PUBLISHING=false if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then # Setting `attestations: true` without Trusted Publishing indicates From d8c894824be9b682f2c96437e9f8002633580706 Mon Sep 17 00:00:00 2001 From: Facundo Tuesca Date: Fri, 27 Sep 2024 20:47:50 +0200 Subject: [PATCH 04/44] Fix magic link nudge formatting in job summary --- twine-upload.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/twine-upload.sh b/twine-upload.sh index 98d41b1b..482e266a 100755 --- a/twine-upload.sh +++ b/twine-upload.sh @@ -69,9 +69,9 @@ The workflow was run with 'attestations: true' input, but the specified \ repository URL does not support PEP 740 attestations. As a result, the \ attestations input is ignored." -MAGIC_LINK_MESSAGE="::warning title=Create a Trusted Publisher::\ -A new Trusted Publisher for the currently running publishing workflow can be created \ -by accessing the following link(s) while logged-in as an owner of the package(s):" +MAGIC_LINK_MESSAGE="A new Trusted Publisher for the currently running \ +publishing workflow can be created by accessing the following link(s) while \ +logged-in as an owner of the package(s):" [[ "${INPUT_USER}" == "__token__" && -z "${INPUT_PASSWORD}" ]] \ @@ -90,10 +90,14 @@ else LINK="- ${INDEX_URL}/manage/project/${PACKAGE_NAME}/settings/publishing/?provider=github&owner=${GITHUB_REPOSITORY_OWNER}&repository=${REPOSITORY_NAME}&workflow_filename=${WORKFLOW_FILENAME}" ALL_LINKS+="$LINK"$'\n' done - TRUSTED_PUBLISHING_MAGIC_LINK_NUDGE="${MAGIC_LINK_MESSAGE}"$'\n'"${ALL_LINKS}" - echo "${MAGIC_LINK_MESSAGE}" >> $GITHUB_STEP_SUMMARY -fi + # Construct the summary message without the warning header + MAGIC_LINK_MESSAGE_WITH_LINKS="${MAGIC_LINK_MESSAGE}"$'\n'"${ALL_LINKS}" + echo "${MAGIC_LINK_MESSAGE_WITH_LINKS}" >> $GITHUB_STEP_SUMMARY + + # The actual nudge in the log is formatted as a warning + TRUSTED_PUBLISHING_MAGIC_LINK_NUDGE="::warning title=Create a Trusted Publisher::${MAGIC_LINK_MESSAGE_WITH_LINKS}" +fi if [[ "${INPUT_ATTESTATIONS}" != "false" ]] ; then # Setting `attestations: true` without Trusted Publishing indicates From 6edc29448558443d6ab3041af8e0140a58f96fba Mon Sep 17 00:00:00 2001 From: mosfet80 Date: Sun, 29 Sep 2024 09:04:41 +0200 Subject: [PATCH 05/44] Fix node.js v16 deprecation self-smoke-test-action.yml actions/checkout@v3 use node.js versio 16. But version 16 is deprecated. version 4 fixes the problem. --- .github/workflows/self-smoke-test-action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/self-smoke-test-action.yml b/.github/workflows/self-smoke-test-action.yml index eecd8cc3..b655019d 100644 --- a/.github/workflows/self-smoke-test-action.yml +++ b/.github/workflows/self-smoke-test-action.yml @@ -42,7 +42,7 @@ jobs: steps: - name: Check out the action locally - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: test - name: Install the packaging-related tools From 1545e96dcbd4dfda3304df772fdf2b616046d32c Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Tue, 22 Oct 2024 12:40:04 -0400 Subject: [PATCH 06/44] requirements: bump sigstore, pypi-attestations Signed-off-by: William Woodruff --- requirements/runtime.in | 4 ++-- requirements/runtime.txt | 8 +++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 3758e3a2..b0a0aaa0 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,8 +10,8 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.12 -sigstore ~= 3.2.0 +pypi-attestations ~= 0.0.13 +sigstore ~= 3.4.0 # NOTE: Used to detect the PyPI package name from the distribution files packaging diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 5ff03bb7..162eb95b 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -72,7 +72,9 @@ pkginfo==1.10.0 platformdirs==4.2.2 # via sigstore pyasn1==0.6.0 - # via sigstore + # via + # pypi-attestations + # sigstore pycparser==2.22 # via cffi pydantic==2.7.1 @@ -91,7 +93,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.12 +pypi-attestations==0.0.13 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto @@ -117,7 +119,7 @@ rich==13.7.1 # twine securesystemslib==1.0.0 # via tuf -sigstore==3.2.0 +sigstore==3.4.0 # via # -r runtime.in # pypi-attestations From 335e8b00ae050a2b9c5938415e7203ca6732f0d5 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 28 Oct 2024 14:29:41 -0400 Subject: [PATCH 07/44] bump sigstore==3.5.1 Signed-off-by: William Woodruff --- requirements/runtime.in | 2 +- requirements/runtime.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index b0a0aaa0..0868f815 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -11,7 +11,7 @@ requests # NOTE: Used to generate attestations. pypi-attestations ~= 0.0.13 -sigstore ~= 3.4.0 +sigstore ~= 3.5.1 # NOTE: Used to detect the PyPI package name from the distribution files packaging diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 162eb95b..13b5f7c0 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -119,7 +119,7 @@ rich==13.7.1 # twine securesystemslib==1.0.0 # via tuf -sigstore==3.4.0 +sigstore==3.5.1 # via # -r runtime.in # pypi-attestations From 0126dcac8e27a0ec77a286e2599ac4a36a3cc2f0 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Mon, 28 Oct 2024 14:31:58 -0400 Subject: [PATCH 08/44] action: enable attestations by default Signed-off-by: William Woodruff --- action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/action.yml b/action.yml index 40fed973..f71598da 100644 --- a/action.yml +++ b/action.yml @@ -86,7 +86,7 @@ inputs: Enable experimental support for PEP 740 attestations. Only works with PyPI and TestPyPI via Trusted Publishing. required: false - default: 'false' + default: 'true' branding: color: yellow icon: upload-cloud From fb13cb306901256ace3dab689990e13a5550ffaa Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 30 Oct 2024 02:20:55 +0100 Subject: [PATCH 09/44] =?UTF-8?q?=F0=9F=93=9D=20Reflect=20the=20PR=20#277?= =?UTF-8?q?=20changes=20in=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This makes minimum modifications to indicate that `attestations` is not on by default. --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index c0998c50..9246fd91 100644 --- a/README.md +++ b/README.md @@ -111,16 +111,17 @@ filter to the job: > Generating and uploading digital attestations currently requires > authentication with a [trusted publisher]. -You can generate signed [digital attestations] for all the distribution files and -upload them all together by enabling the `attestations` setting: +Generating signed [digital attestations] for all the distribution files +and uploading them all together is now on by default for all projects +using Trusted Publishing. To disable it, set `attestations` as follows: ```yml with: - attestations: true + attestations: false ``` -This will use [Sigstore] to create attestation -objects for each distribution package, signing them with the identity provided +The attestation objects are created using [Sigstore] for each +distribution package, signing them with the identity provided by the GitHub's OIDC token associated with the current workflow. This means both the trusted publishing authentication and the attestations are tied to the same identity. From 783267be6946bcf454fcec01dd3ddf45c7830dbc Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Tue, 16 Apr 2024 03:02:03 -0400 Subject: [PATCH 10/44] Build Docker image and push to GHCR Up to this point, the project has been set up as a Docker action referencing the Dockerfile. The downside to using the Dockerfile for the action is that the Docker image must be built every time the action is used. This commit will set up the project to build the Docker image and push it to GitHub Container Registry (GHCR). This change will speed up user workflows every time the action is used because the workflows will simply pull the Docker image from GHCR instead of building again. Changes: - Add required metadata to Dockerfile - Build container image with GitHub Actions - Push container image to GHCR Docker actions support pulling in pre-built Docker images. The downside is that there's no way to specify the correct Docker tag because the GitHub Actions `image` and `uses:` keys don't accept any context. For example, if a user's workflow has `uses: pypa/gh-action-pypi-publish@release/v1.8`, then the action should pull in a Docker image built from the `release/v1.8` branch, something like `ghcr.io/pypa/gh-action-pypi-publish:release-v1.8` (Docker tags can't have `/`). The workaround is to switch the top-level `action.yml` to a composite action that then calls the Docker action, substituting the correct image name and tag. --- .../actions/run-docker-container/action.yml | 32 +++++++++ .../workflows/build-and-push-docker-image.yml | 29 ++++++++ .github/workflows/self-smoke-test-action.yml | 7 +- action.yml | 69 +++++++++++++++---- 4 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 .github/actions/run-docker-container/action.yml create mode 100644 .github/workflows/build-and-push-docker-image.yml diff --git a/.github/actions/run-docker-container/action.yml b/.github/actions/run-docker-container/action.yml new file mode 100644 index 00000000..05a04b13 --- /dev/null +++ b/.github/actions/run-docker-container/action.yml @@ -0,0 +1,32 @@ +--- +name: 🏃 +inputs: + user: + required: false + password: + required: false + repository-url: + required: false + packages-dir: + required: false + verify-metadata: + required: false + skip-existing: + required: false + verbose: + required: false + print-hash: + required: false +runs: + using: docker + image: {{image}} + args: + - ${{ inputs.user }} + - ${{ inputs.password }} + - ${{ inputs.repository-url }} + - ${{ inputs.packages-dir }} + - ${{ inputs.verify-metadata }} + - ${{ inputs.skip-existing }} + - ${{ inputs.verbose }} + - ${{ inputs.print-hash }} + - ${{ inputs.attestations }} diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml new file mode 100644 index 00000000..cd3c7319 --- /dev/null +++ b/.github/workflows/build-and-push-docker-image.yml @@ -0,0 +1,29 @@ +--- + +name: 🏗️ + +on: # yamllint disable-line rule:truthy + pull_request: + push: + branches: ["release/*", "unstable/*"] + tags: ["*"] + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build Docker image + run: | + IMAGE="ghcr.io/$GITHUB_REPOSITORY:${GITHUB_REF_NAME/'/'/'-'}" + echo "IMAGE=$IMAGE" >>"$GITHUB_ENV" + docker build . \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from $IMAGE \ + --tag $IMAGE + - name: Push Docker image to GHCR + if: github.event_name != 'pull_request' + run: | + echo ${{ secrets.GITHUB_TOKEN }} | + docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + docker push $IMAGE diff --git a/.github/workflows/self-smoke-test-action.yml b/.github/workflows/self-smoke-test-action.yml index b655019d..0a88c1cd 100644 --- a/.github/workflows/self-smoke-test-action.yml +++ b/.github/workflows/self-smoke-test-action.yml @@ -3,8 +3,10 @@ name: 🧪 on: # yamllint disable-line rule:truthy - push: pull_request: + workflow_run: + workflows: [🏗️] + types: [completed] env: devpi-password: abcd1234 @@ -28,6 +30,9 @@ env: jobs: smoke-test: + if: >- + github.event_name == 'pull_request' || + github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest services: diff --git a/action.yml b/action.yml index f71598da..e67f9c2f 100644 --- a/action.yml +++ b/action.yml @@ -91,15 +91,60 @@ branding: color: yellow icon: upload-cloud runs: - using: docker - image: Dockerfile - args: - - ${{ inputs.user }} - - ${{ inputs.password }} - - ${{ inputs.repository-url }} - - ${{ inputs.packages-dir }} - - ${{ inputs.verify-metadata }} - - ${{ inputs.skip-existing }} - - ${{ inputs.verbose }} - - ${{ inputs.print-hash }} - - ${{ inputs.attestations }} + using: composite + steps: + - name: Reset path if needed + run: | + # Reset path if needed + # https://github.com/pypa/gh-action-pypi-publish/issues/112 + if [[ $PATH != *"/usr/bin"* ]]; then + echo "\$PATH=$PATH. Resetting \$PATH for GitHub Actions." + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + echo "PATH=$PATH" >>"$GITHUB_ENV" + echo "$PATH" >>"$GITHUB_PATH" + echo "\$PATH reset. \$PATH=$PATH" + fi + shell: bash + - name: Set repo and ref from which to run Docker container action + id: set-repo-and-ref + run: | + # Set repo and ref from which to run Docker container action + # to handle cases in which `github.action_` context is not set + # https://github.com/actions/runner/issues/2473 + REF=${{ env.ACTION_REF || github.ref_name }} + REPO=${{ env.ACTION_REPO || github.repository }} + echo "ref=$REF" >>"$GITHUB_OUTPUT" + echo "repo=$REPO" >>"$GITHUB_OUTPUT" + shell: bash + env: + ACTION_REF: ${{ github.action_ref }} + ACTION_REPO: ${{ github.action_repository }} + - name: Set Docker image name and tag + run: | + # Set Docker image name and tag + # if action run was triggered by a pull request to this repo, + # build image from Dockerfile because it has not been pushed to GHCR, + # else pull image from GHCR + if [[ $GITHUB_EVENT_NAME == "pull_request" ]] && + [[ $GITHUB_REPOSITORY == "pypa/gh-action-pypi-publish" ]]; then + IMAGE="../../../Dockerfile" + else + REF=${{ steps.set-repo-and-ref.outputs.ref }} + REPO=${{ steps.set-repo-and-ref.outputs.repo }} + IMAGE="docker://ghcr.io/$REPO:${REF/'/'/'-'}" + fi + FILE=".github/actions/run-docker-container/action.yml" + sed -i -e "s|{{image}}|$IMAGE|g" "$FILE" + shell: bash + - name: Run Docker container + uses: ./.github/actions/run-docker-container + with: + user: ${{ inputs.user }} + password: ${{ inputs.password }} + repository-url: ${{ inputs.repository-url || inputs.repository_url }} + packages-dir: ${{ inputs.packages-dir || inputs.packages_dir }} + verify-metadata: ${{ inputs.verify-metadata || inputs.verify_metadata }} + skip-existing: ${{ inputs.skip-existing || inputs.skip_existing }} + verbose: ${{ inputs.verbose }} + print-hash: ${{ inputs.print-hash || inputs.print_hash }} + attestations: ${{ inputs.attestations }} From e453f8c630387210460e03d303395b75b19f3b99 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Thu, 18 Apr 2024 22:01:29 -0400 Subject: [PATCH 11/44] Fix pre-commit errors --- .github/actions/run-docker-container/action.yml | 15 ++++++++++++++- .github/workflows/build-and-push-docker-image.yml | 1 + .pre-commit-config.yaml | 2 +- 3 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-docker-container/action.yml b/.github/actions/run-docker-container/action.yml index 05a04b13..e1b1ecd2 100644 --- a/.github/actions/run-docker-container/action.yml +++ b/.github/actions/run-docker-container/action.yml @@ -1,25 +1,38 @@ --- name: 🏃 +description: >- + Run Docker container to + upload Python distribution packages to PyPI inputs: user: + description: PyPI user required: false password: + description: Password for your PyPI user or an access token required: false repository-url: + description: The repository URL to use required: false packages-dir: + description: The target directory for distribution required: false verify-metadata: + description: Check metadata before uploading required: false skip-existing: + description: >- + Do not fail if a Python package distribution + exists in the target package index required: false verbose: + description: Show verbose output. required: false print-hash: + description: Show hash values of files to be uploaded required: false runs: using: docker - image: {{image}} + image: "{{image}}" args: - ${{ inputs.user }} - ${{ inputs.password }} diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index cd3c7319..57980feb 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -11,6 +11,7 @@ on: # yamllint disable-line rule:truthy jobs: build-and-push: runs-on: ubuntu-latest + timeout-minutes: 10 steps: - uses: actions/checkout@v4 - name: Build Docker image diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d5f5f7e6..6f1ea806 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: args: - --builtin-schema - github-workflows-require-timeout - files: ^\.github/workflows/[^/]+$ + files: ^\.github\/workflows/[^/]+$ types: - yaml - id: check-readthedocs From 0d8d5059c85abef0befd45626d6d08c98a8a3680 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Fri, 26 Apr 2024 19:08:30 -0400 Subject: [PATCH 12/44] Separate `docker login` and `docker push` https://github.com/pypa/gh-action-pypi-publish/pull/230#discussion_r1578694138 --- .github/workflows/build-and-push-docker-image.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index 57980feb..d8112676 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -22,9 +22,12 @@ jobs: --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from $IMAGE \ --tag $IMAGE - - name: Push Docker image to GHCR + - name: Log in to GHCR if: github.event_name != 'pull_request' run: | echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + - name: Push Docker image to GHCR + if: github.event_name != 'pull_request' + run: | docker push $IMAGE From aed6c4b1b02d8644667434f8e4c077b9a2db90b4 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Mon, 27 May 2024 13:45:20 -0400 Subject: [PATCH 13/44] Generate Docker container action with Python --- .../actions/run-docker-container/action.yml | 45 ---------- action.yml | 32 +++---- create-docker-action.py | 88 +++++++++++++++++++ requirements/github-actions.in | 2 + requirements/github-actions.txt | 8 ++ 5 files changed, 114 insertions(+), 61 deletions(-) delete mode 100644 .github/actions/run-docker-container/action.yml create mode 100644 create-docker-action.py create mode 100644 requirements/github-actions.in create mode 100644 requirements/github-actions.txt diff --git a/.github/actions/run-docker-container/action.yml b/.github/actions/run-docker-container/action.yml deleted file mode 100644 index e1b1ecd2..00000000 --- a/.github/actions/run-docker-container/action.yml +++ /dev/null @@ -1,45 +0,0 @@ ---- -name: 🏃 -description: >- - Run Docker container to - upload Python distribution packages to PyPI -inputs: - user: - description: PyPI user - required: false - password: - description: Password for your PyPI user or an access token - required: false - repository-url: - description: The repository URL to use - required: false - packages-dir: - description: The target directory for distribution - required: false - verify-metadata: - description: Check metadata before uploading - required: false - skip-existing: - description: >- - Do not fail if a Python package distribution - exists in the target package index - required: false - verbose: - description: Show verbose output. - required: false - print-hash: - description: Show hash values of files to be uploaded - required: false -runs: - using: docker - image: "{{image}}" - args: - - ${{ inputs.user }} - - ${{ inputs.password }} - - ${{ inputs.repository-url }} - - ${{ inputs.packages-dir }} - - ${{ inputs.verify-metadata }} - - ${{ inputs.skip-existing }} - - ${{ inputs.verbose }} - - ${{ inputs.print-hash }} - - ${{ inputs.attestations }} diff --git a/action.yml b/action.yml index e67f9c2f..d051ab12 100644 --- a/action.yml +++ b/action.yml @@ -119,25 +119,25 @@ runs: env: ACTION_REF: ${{ github.action_ref }} ACTION_REPO: ${{ github.action_repository }} - - name: Set Docker image name and tag + - name: Check out action repo + uses: actions/checkout@v4 + with: + path: action-repo + ref: ${{ steps.set-repo-and-ref.outputs.ref }} + repository: ${{ steps.set-repo-and-ref.outputs.repo }} + - name: Create Docker container action run: | - # Set Docker image name and tag - # if action run was triggered by a pull request to this repo, - # build image from Dockerfile because it has not been pushed to GHCR, - # else pull image from GHCR - if [[ $GITHUB_EVENT_NAME == "pull_request" ]] && - [[ $GITHUB_REPOSITORY == "pypa/gh-action-pypi-publish" ]]; then - IMAGE="../../../Dockerfile" - else - REF=${{ steps.set-repo-and-ref.outputs.ref }} - REPO=${{ steps.set-repo-and-ref.outputs.repo }} - IMAGE="docker://ghcr.io/$REPO:${REF/'/'/'-'}" - fi - FILE=".github/actions/run-docker-container/action.yml" - sed -i -e "s|{{image}}|$IMAGE|g" "$FILE" + # Create Docker container action + python -m pip install -r requirements/github-actions.txt + python create-docker-action.py ${{ steps.set-image.outputs.image }} + env: + EVENT: ${{ github.event_name }} + REF: ${{ steps.set-repo-and-ref.outputs.ref }} + REPO: ${{ steps.set-repo-and-ref.outputs.repo }} shell: bash + working-directory: action-repo - name: Run Docker container - uses: ./.github/actions/run-docker-container + uses: ./action-repo/.github/actions/run-docker-container with: user: ${{ inputs.user }} password: ${{ inputs.password }} diff --git a/create-docker-action.py b/create-docker-action.py new file mode 100644 index 00000000..bc2e0052 --- /dev/null +++ b/create-docker-action.py @@ -0,0 +1,88 @@ +import os +import pathlib + +import yaml + +DESCRIPTION = 'description' +REQUIRED = 'required' + +EVENT = os.environ['EVENT'] +REF = os.environ['REF'] +REPO = os.environ['REPO'] + + +def set_image(event: str, ref: str, repo: str) -> str: + if event == 'pull_request' and 'gh-action-pypi-publish' in repo: + return '../../../Dockerfile' + docker_ref = ref.replace('/', '-') + return f'docker://ghcr.io/{repo}:{docker_ref}' + + +image = set_image(EVENT, REF, REPO) + +action = { + 'name': '🏃', + DESCRIPTION: ( + 'Run Docker container to upload Python distribution packages to PyPI' + ), + 'inputs': { + 'user': {DESCRIPTION: 'PyPI user', REQUIRED: False}, + 'password': { + DESCRIPTION: 'Password for your PyPI user or an access token', + REQUIRED: False, + }, + 'repository-url': { + DESCRIPTION: 'The repository URL to use', + REQUIRED: False, + }, + 'packages-dir': { + DESCRIPTION: 'The target directory for distribution', + REQUIRED: False, + }, + 'verify-metadata': { + DESCRIPTION: 'Check metadata before uploading', + REQUIRED: False, + }, + 'skip-existing': { + DESCRIPTION: ( + 'Do not fail if a Python package distribution' + ' exists in the target package index' + ), + REQUIRED: False, + }, + 'verbose': {DESCRIPTION: 'Show verbose output.', REQUIRED: False}, + 'print-hash': { + DESCRIPTION: 'Show hash values of files to be uploaded', + REQUIRED: False, + }, + 'attestations': { + DESCRIPTION: ( + '[EXPERIMENTAL]' + ' Enable experimental support for PEP 740 attestations.' + ' Only works with PyPI and TestPyPI via Trusted Publishing.' + ), + REQUIRED: False, + } + }, + 'runs': { + 'using': 'docker', + 'image': image, + 'args': [ + '${{ inputs.user }}', + '${{ inputs.password }}', + '${{ inputs.repository-url }}', + '${{ inputs.packages-dir }}', + '${{ inputs.verify-metadata }}', + '${{ inputs.skip-existing }}', + '${{ inputs.verbose }}', + '${{ inputs.print-hash }}', + '${{ inputs.attestations }}', + ], + }, +} + +action_path = pathlib.Path('.github/actions/run-docker-container/action.yml') +action_path.parent.mkdir(parents=True, exist_ok=True) + +with action_path.open(mode='w', encoding='utf-8') as file: + yaml.dump(action, file, allow_unicode=True, sort_keys=False) diff --git a/requirements/github-actions.in b/requirements/github-actions.in new file mode 100644 index 00000000..23147228 --- /dev/null +++ b/requirements/github-actions.in @@ -0,0 +1,2 @@ +# NOTE: used by create-docker-action.py +pyyaml diff --git a/requirements/github-actions.txt b/requirements/github-actions.txt new file mode 100644 index 00000000..0f1caa21 --- /dev/null +++ b/requirements/github-actions.txt @@ -0,0 +1,8 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --config=../.pip-tools.toml --output-file=github-actions.txt --strip-extras github-actions.in +# +pyyaml==6.0.1 + # via -r github-actions.in From f1f014b445fb2d36e5923f270268cc003dedc729 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Mon, 27 May 2024 14:08:24 -0400 Subject: [PATCH 14/44] Reset pre-commit `files:` regex --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6f1ea806..d5f5f7e6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: args: - --builtin-schema - github-workflows-require-timeout - files: ^\.github\/workflows/[^/]+$ + files: ^\.github/workflows/[^/]+$ types: - yaml - id: check-readthedocs From cf5ce177daab78038d84aabd1e53da2b26e621f1 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Mon, 27 May 2024 14:00:48 -0400 Subject: [PATCH 15/44] Use YAML block strip syntax (`>-`) where possible --- .github/workflows/build-and-push-docker-image.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index d8112676..417891c9 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -24,10 +24,9 @@ jobs: --tag $IMAGE - name: Log in to GHCR if: github.event_name != 'pull_request' - run: | + run: >- echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin - name: Push Docker image to GHCR if: github.event_name != 'pull_request' - run: | - docker push $IMAGE + run: docker push $IMAGE From 5ded5310e7365c2319d8cefbe17deb18c87cec74 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Mon, 27 May 2024 14:39:55 -0400 Subject: [PATCH 16/44] Add `workflow_dispatch` trigger for Docker builds --- .github/workflows/build-and-push-docker-image.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index 417891c9..f39a715a 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -6,7 +6,12 @@ on: # yamllint disable-line rule:truthy pull_request: push: branches: ["release/*", "unstable/*"] - tags: ["*"] + workflow_dispatch: + inputs: + tag: + description: Docker image tag + required: true + type: string jobs: build-and-push: @@ -16,12 +21,14 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker image run: | - IMAGE="ghcr.io/$GITHUB_REPOSITORY:${GITHUB_REF_NAME/'/'/'-'}" + IMAGE="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG/'/'/'-'}" echo "IMAGE=$IMAGE" >>"$GITHUB_ENV" docker build . \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from $IMAGE \ --tag $IMAGE + env: + DOCKER_TAG: ${{ inputs.tag || github.ref_name }} - name: Log in to GHCR if: github.event_name != 'pull_request' run: >- From a869dd36b2474e3145d15d4d13013b7622bc12b0 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Thu, 30 May 2024 21:34:34 -0400 Subject: [PATCH 17/44] Checkout `github.head_ref` and repo for PRs https://github.com/actions/checkout/issues/27#issuecomment-535897113 https://github.com/actions/checkout/issues/1108 --- action.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/action.yml b/action.yml index d051ab12..2ae5a8e5 100644 --- a/action.yml +++ b/action.yml @@ -111,14 +111,16 @@ runs: # Set repo and ref from which to run Docker container action # to handle cases in which `github.action_` context is not set # https://github.com/actions/runner/issues/2473 - REF=${{ env.ACTION_REF || github.ref_name }} - REPO=${{ env.ACTION_REPO || github.repository }} + REF=${{ env.ACTION_REF || env.PR_REF || github.ref_name }} + REPO=${{ env.ACTION_REPO || env.PR_REPO || github.repository }} echo "ref=$REF" >>"$GITHUB_OUTPUT" echo "repo=$REPO" >>"$GITHUB_OUTPUT" shell: bash env: ACTION_REF: ${{ github.action_ref }} ACTION_REPO: ${{ github.action_repository }} + PR_REF: ${{ github.event.pull_request.head.ref }} + PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} - name: Check out action repo uses: actions/checkout@v4 with: From a360fcb184bd4cefddaef3d4bbdf179053dfcb3e Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 1 Jun 2024 17:08:20 -0400 Subject: [PATCH 18/44] Dump action as JSON --- action.yml | 3 +-- create-docker-action.py | 7 ++----- requirements/github-actions.in | 2 -- requirements/github-actions.txt | 8 -------- 4 files changed, 3 insertions(+), 17 deletions(-) delete mode 100644 requirements/github-actions.in delete mode 100644 requirements/github-actions.txt diff --git a/action.yml b/action.yml index 2ae5a8e5..ea0a4d2c 100644 --- a/action.yml +++ b/action.yml @@ -130,8 +130,7 @@ runs: - name: Create Docker container action run: | # Create Docker container action - python -m pip install -r requirements/github-actions.txt - python create-docker-action.py ${{ steps.set-image.outputs.image }} + python create-docker-action.py env: EVENT: ${{ github.event_name }} REF: ${{ steps.set-repo-and-ref.outputs.ref }} diff --git a/create-docker-action.py b/create-docker-action.py index bc2e0052..dc361559 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -1,8 +1,7 @@ +import json import os import pathlib -import yaml - DESCRIPTION = 'description' REQUIRED = 'required' @@ -83,6 +82,4 @@ def set_image(event: str, ref: str, repo: str) -> str: action_path = pathlib.Path('.github/actions/run-docker-container/action.yml') action_path.parent.mkdir(parents=True, exist_ok=True) - -with action_path.open(mode='w', encoding='utf-8') as file: - yaml.dump(action, file, allow_unicode=True, sort_keys=False) +action_path.write_text(json.dumps(action, ensure_ascii=False), encoding='utf-8') diff --git a/requirements/github-actions.in b/requirements/github-actions.in deleted file mode 100644 index 23147228..00000000 --- a/requirements/github-actions.in +++ /dev/null @@ -1,2 +0,0 @@ -# NOTE: used by create-docker-action.py -pyyaml diff --git a/requirements/github-actions.txt b/requirements/github-actions.txt deleted file mode 100644 index 0f1caa21..00000000 --- a/requirements/github-actions.txt +++ /dev/null @@ -1,8 +0,0 @@ -# -# This file is autogenerated by pip-compile with Python 3.12 -# by the following command: -# -# pip-compile --allow-unsafe --config=../.pip-tools.toml --output-file=github-actions.txt --strip-extras github-actions.in -# -pyyaml==6.0.1 - # via -r github-actions.in From f51682fb52eebc08c0cacb1386f97c89eff75cb3 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sun, 9 Jun 2024 16:47:19 -0400 Subject: [PATCH 19/44] Check repo owner ID instead of repo name --- action.yml | 4 ++++ create-docker-action.py | 8 +++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/action.yml b/action.yml index ea0a4d2c..76e8eb68 100644 --- a/action.yml +++ b/action.yml @@ -113,14 +113,17 @@ runs: # https://github.com/actions/runner/issues/2473 REF=${{ env.ACTION_REF || env.PR_REF || github.ref_name }} REPO=${{ env.ACTION_REPO || env.PR_REPO || github.repository }} + REPO_OWNER_ID=${{ env.PR_REPO_OWNER_ID || github.repository_owner_id }} echo "ref=$REF" >>"$GITHUB_OUTPUT" echo "repo=$REPO" >>"$GITHUB_OUTPUT" + echo "repo-owner-id=$REPO_OWNER_ID" >>"$GITHUB_OUTPUT" shell: bash env: ACTION_REF: ${{ github.action_ref }} ACTION_REPO: ${{ github.action_repository }} PR_REF: ${{ github.event.pull_request.head.ref }} PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} + PR_REPO_OWNER_ID: ${{ github.event.pull_request.base.repo.owner.id }} - name: Check out action repo uses: actions/checkout@v4 with: @@ -135,6 +138,7 @@ runs: EVENT: ${{ github.event_name }} REF: ${{ steps.set-repo-and-ref.outputs.ref }} REPO: ${{ steps.set-repo-and-ref.outputs.repo }} + REPO_OWNER_ID: ${{ steps.set-repo-and-ref.outputs.repo-owner-id }} shell: bash working-directory: action-repo - name: Run Docker container diff --git a/create-docker-action.py b/create-docker-action.py index dc361559..a51d27df 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -8,16 +8,18 @@ EVENT = os.environ['EVENT'] REF = os.environ['REF'] REPO = os.environ['REPO'] +REPO_OWNER_ID = os.environ['REPO_OWNER_ID'] +REPO_OWNER_ID_PYPA = '647025' -def set_image(event: str, ref: str, repo: str) -> str: - if event == 'pull_request' and 'gh-action-pypi-publish' in repo: +def set_image(event: str, ref: str, repo: str, repo_owner_id: str) -> str: + if event == 'pull_request' and repo_owner_id == REPO_OWNER_ID_PYPA: return '../../../Dockerfile' docker_ref = ref.replace('/', '-') return f'docker://ghcr.io/{repo}:{docker_ref}' -image = set_image(EVENT, REF, REPO) +image = set_image(EVENT, REF, REPO, REPO_OWNER_ID) action = { 'name': '🏃', From 7ea8313fc2c8774b7904a970426f44c5a8defbcc Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sun, 9 Jun 2024 20:08:19 -0400 Subject: [PATCH 20/44] Check repo ID instead of repo owner ID --- action.yml | 8 ++++---- create-docker-action.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/action.yml b/action.yml index 76e8eb68..7320b869 100644 --- a/action.yml +++ b/action.yml @@ -113,17 +113,17 @@ runs: # https://github.com/actions/runner/issues/2473 REF=${{ env.ACTION_REF || env.PR_REF || github.ref_name }} REPO=${{ env.ACTION_REPO || env.PR_REPO || github.repository }} - REPO_OWNER_ID=${{ env.PR_REPO_OWNER_ID || github.repository_owner_id }} + REPO_ID=${{ env.PR_REPO_ID || github.repository_id }} echo "ref=$REF" >>"$GITHUB_OUTPUT" echo "repo=$REPO" >>"$GITHUB_OUTPUT" - echo "repo-owner-id=$REPO_OWNER_ID" >>"$GITHUB_OUTPUT" + echo "repo-id=$REPO_ID" >>"$GITHUB_OUTPUT" shell: bash env: ACTION_REF: ${{ github.action_ref }} ACTION_REPO: ${{ github.action_repository }} PR_REF: ${{ github.event.pull_request.head.ref }} PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} - PR_REPO_OWNER_ID: ${{ github.event.pull_request.base.repo.owner.id }} + PR_REPO_ID: ${{ github.event.pull_request.base.repo.id }} - name: Check out action repo uses: actions/checkout@v4 with: @@ -138,7 +138,7 @@ runs: EVENT: ${{ github.event_name }} REF: ${{ steps.set-repo-and-ref.outputs.ref }} REPO: ${{ steps.set-repo-and-ref.outputs.repo }} - REPO_OWNER_ID: ${{ steps.set-repo-and-ref.outputs.repo-owner-id }} + REPO_ID: ${{ steps.set-repo-and-ref.outputs.repo-id }} shell: bash working-directory: action-repo - name: Run Docker container diff --git a/create-docker-action.py b/create-docker-action.py index a51d27df..86adb19f 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -8,18 +8,18 @@ EVENT = os.environ['EVENT'] REF = os.environ['REF'] REPO = os.environ['REPO'] -REPO_OWNER_ID = os.environ['REPO_OWNER_ID'] -REPO_OWNER_ID_PYPA = '647025' +REPO_ID = os.environ['REPO_ID'] +REPO_ID_GH_ACTION = '178055147' -def set_image(event: str, ref: str, repo: str, repo_owner_id: str) -> str: - if event == 'pull_request' and repo_owner_id == REPO_OWNER_ID_PYPA: +def set_image(event: str, ref: str, repo: str, repo_id: str) -> str: + if event == 'pull_request' and repo_id == REPO_ID_GH_ACTION: return '../../../Dockerfile' docker_ref = ref.replace('/', '-') return f'docker://ghcr.io/{repo}:{docker_ref}' -image = set_image(EVENT, REF, REPO, REPO_OWNER_ID) +image = set_image(EVENT, REF, REPO, REPO_ID) action = { 'name': '🏃', From bacb62682c754871d3471af8890de22d03b01199 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Tue, 11 Jun 2024 12:53:35 -0400 Subject: [PATCH 21/44] Fail-fast in unsupported environments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit https://github.com/pypa/gh-action-pypi-publish/pull/230#discussion_r1632406604 Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/action.yml b/action.yml index 7320b869..508d4c61 100644 --- a/action.yml +++ b/action.yml @@ -93,6 +93,12 @@ branding: runs: using: composite steps: + - name: Fail-fast in unsupported environments + if: runner.os != 'Linux' + run: | + >&2 echo This action is only able to run under GNU/Linux environments + exit 1 + shell: bash -eEuo pipefail {0} - name: Reset path if needed run: | # Reset path if needed From d03addb8e640d5bc0f05e40dfc630b3fa59776d8 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Tue, 11 Jun 2024 13:01:29 -0400 Subject: [PATCH 22/44] Drop args from create-docker-action.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Sviatoslav Sydorenko (Святослав Сидоренко) --- create-docker-action.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/create-docker-action.py b/create-docker-action.py index 86adb19f..cf3eb5f3 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -68,17 +68,6 @@ def set_image(event: str, ref: str, repo: str, repo_id: str) -> str: 'runs': { 'using': 'docker', 'image': image, - 'args': [ - '${{ inputs.user }}', - '${{ inputs.password }}', - '${{ inputs.repository-url }}', - '${{ inputs.packages-dir }}', - '${{ inputs.verify-metadata }}', - '${{ inputs.skip-existing }}', - '${{ inputs.verbose }}', - '${{ inputs.print-hash }}', - '${{ inputs.attestations }}', - ], }, } From 153ccde9bc89de7fabd26026009f28e4c3f5bae1 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Wed, 10 Jul 2024 18:40:53 -0400 Subject: [PATCH 23/44] Verify fail-fast in unsupported environments --- .github/workflows/self-smoke-test-action.yml | 28 ++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/self-smoke-test-action.yml b/.github/workflows/self-smoke-test-action.yml index 0a88c1cd..ddddf832 100644 --- a/.github/workflows/self-smoke-test-action.yml +++ b/.github/workflows/self-smoke-test-action.yml @@ -29,6 +29,34 @@ env: PYTEST_THEME_MODE jobs: + fail-fast: + if: >- + github.event_name == 'pull_request' || + github.event.workflow_run.conclusion == 'success' + + strategy: + matrix: + os: [macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + timeout-minutes: 2 + + steps: + - name: Check out the action locally + uses: actions/checkout@v3 + with: + path: test + - name: Fail-fast in unsupported environments + continue-on-error: true + id: fail-fast + uses: ./test + - name: Error if action did not fail-fast in unsupported environments + if: steps.fail-fast.outcome == 'success' + run: | + >&2 echo This action should fail-fast in unsupported environments. + exit 1 + smoke-test: if: >- github.event_name == 'pull_request' || From cfb9d93a2623522535b927826fc87684f70365d6 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 7 Sep 2024 14:28:31 -0400 Subject: [PATCH 24/44] Add Docker tags for major and minor versions --- .../workflows/build-and-push-docker-image.yml | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index f39a715a..39f3163d 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -9,8 +9,8 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: tag: - description: Docker image tag - required: true + description: Docker image tag (optional, defaults to Git ref) + required: false type: string jobs: @@ -21,12 +21,21 @@ jobs: - uses: actions/checkout@v4 - name: Build Docker image run: | - IMAGE="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG/'/'/'-'}" + DOCKER_TAG="${DOCKER_TAG/'/'/'-'}" + DOCKER_TAG_MAJOR=$(echo "$DOCKER_TAG" | cut -d '.' -f 1) + DOCKER_TAG_MAJOR_MINOR=$(echo "$DOCKER_TAG" | cut -d '.' -f 1-2) + IMAGE="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG}" + IMAGE_MAJOR="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG_MAJOR}" + IMAGE_MAJOR_MINOR="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG_MAJOR_MINOR}" echo "IMAGE=$IMAGE" >>"$GITHUB_ENV" + echo "IMAGE_MAJOR=$IMAGE_MAJOR" >>"$GITHUB_ENV" + echo "IMAGE_MAJOR_MINOR=$IMAGE_MAJOR_MINOR" >>"$GITHUB_ENV" docker build . \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from $IMAGE \ --tag $IMAGE + docker tag $IMAGE $IMAGE_MAJOR + docker tag $IMAGE $IMAGE_MAJOR_MINOR env: DOCKER_TAG: ${{ inputs.tag || github.ref_name }} - name: Log in to GHCR @@ -36,4 +45,7 @@ jobs: docker login ghcr.io -u $GITHUB_ACTOR --password-stdin - name: Push Docker image to GHCR if: github.event_name != 'pull_request' - run: docker push $IMAGE + run: | + docker push $IMAGE + docker push $IMAGE_MAJOR + docker push $IMAGE_MAJOR_MINOR From 1b9f21a74186d118c2657dfb4595b0526ecde1a8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 7 Sep 2024 18:39:31 +0000 Subject: [PATCH 25/44] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- create-docker-action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create-docker-action.py b/create-docker-action.py index cf3eb5f3..7b42ad81 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -63,7 +63,7 @@ def set_image(event: str, ref: str, repo: str, repo_id: str) -> str: ' Only works with PyPI and TestPyPI via Trusted Publishing.' ), REQUIRED: False, - } + }, }, 'runs': { 'using': 'docker', From 80b1d50e0dd4d304b48449d781099aee65e1ee35 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Sat, 14 Sep 2024 14:05:40 -0400 Subject: [PATCH 26/44] Make `workflow_dispatch` Docker tag input required https://github.com/pypa/gh-action-pypi-publish/pull/230#discussion_r1759496153 --- .github/workflows/build-and-push-docker-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index 39f3163d..f8e9ae5c 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -9,8 +9,8 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: tag: - description: Docker image tag (optional, defaults to Git ref) - required: false + description: Docker image tag + required: true type: string jobs: From da554410b00985409d772ab68bf845b44bbf5f4c Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Fri, 4 Oct 2024 18:20:25 -0400 Subject: [PATCH 27/44] Move smoke test to reusable workflow --- .../workflows/build-and-push-docker-image.yml | 4 ++++ ...e-test-action.yml => reusable-smoke-test.yml} | 16 ++++------------ 2 files changed, 8 insertions(+), 12 deletions(-) rename .github/workflows/{self-smoke-test-action.yml => reusable-smoke-test.yml} (90%) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index f8e9ae5c..fda55e58 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -49,3 +49,7 @@ jobs: docker push $IMAGE docker push $IMAGE_MAJOR docker push $IMAGE_MAJOR_MINOR + smoke-test: + needs: + - build-and-push + uses: ./.github/workflows/reusable-smoke-test.yml diff --git a/.github/workflows/self-smoke-test-action.yml b/.github/workflows/reusable-smoke-test.yml similarity index 90% rename from .github/workflows/self-smoke-test-action.yml rename to .github/workflows/reusable-smoke-test.yml index ddddf832..ac59f08f 100644 --- a/.github/workflows/self-smoke-test-action.yml +++ b/.github/workflows/reusable-smoke-test.yml @@ -1,12 +1,9 @@ --- -name: 🧪 +name: ♻️ 🧪 on: # yamllint disable-line rule:truthy - pull_request: - workflow_run: - workflows: [🏗️] - types: [completed] + workflow_call: env: devpi-password: abcd1234 @@ -30,9 +27,6 @@ env: jobs: fail-fast: - if: >- - github.event_name == 'pull_request' || - github.event.workflow_run.conclusion == 'success' strategy: matrix: @@ -44,7 +38,7 @@ jobs: steps: - name: Check out the action locally - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: path: test - name: Fail-fast in unsupported environments @@ -58,9 +52,7 @@ jobs: exit 1 smoke-test: - if: >- - github.event_name == 'pull_request' || - github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest services: From 36965cb24aed76043aeb8f19062b6a31fa56b990 Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Fri, 4 Oct 2024 18:23:49 -0400 Subject: [PATCH 28/44] Run smoke tests before Docker builds https://github.com/pypa/gh-action-pypi-publish/pull/230#discussion_r1787027821 --- .github/workflows/build-and-push-docker-image.yml | 11 +++++------ action.yml | 1 - create-docker-action.py | 7 +++---- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index fda55e58..d8fd716c 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -14,8 +14,13 @@ on: # yamllint disable-line rule:truthy type: string jobs: + smoke-test: + uses: ./.github/workflows/reusable-smoke-test.yml build-and-push: + if: github.event_name != 'pull_request' runs-on: ubuntu-latest + needs: + - smoke-test timeout-minutes: 10 steps: - uses: actions/checkout@v4 @@ -39,17 +44,11 @@ jobs: env: DOCKER_TAG: ${{ inputs.tag || github.ref_name }} - name: Log in to GHCR - if: github.event_name != 'pull_request' run: >- echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin - name: Push Docker image to GHCR - if: github.event_name != 'pull_request' run: | docker push $IMAGE docker push $IMAGE_MAJOR docker push $IMAGE_MAJOR_MINOR - smoke-test: - needs: - - build-and-push - uses: ./.github/workflows/reusable-smoke-test.yml diff --git a/action.yml b/action.yml index 508d4c61..7f6b7461 100644 --- a/action.yml +++ b/action.yml @@ -141,7 +141,6 @@ runs: # Create Docker container action python create-docker-action.py env: - EVENT: ${{ github.event_name }} REF: ${{ steps.set-repo-and-ref.outputs.ref }} REPO: ${{ steps.set-repo-and-ref.outputs.repo }} REPO_ID: ${{ steps.set-repo-and-ref.outputs.repo-id }} diff --git a/create-docker-action.py b/create-docker-action.py index 7b42ad81..16aa54c0 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -5,21 +5,20 @@ DESCRIPTION = 'description' REQUIRED = 'required' -EVENT = os.environ['EVENT'] REF = os.environ['REF'] REPO = os.environ['REPO'] REPO_ID = os.environ['REPO_ID'] REPO_ID_GH_ACTION = '178055147' -def set_image(event: str, ref: str, repo: str, repo_id: str) -> str: - if event == 'pull_request' and repo_id == REPO_ID_GH_ACTION: +def set_image(ref: str, repo: str, repo_id: str) -> str: + if repo_id == REPO_ID_GH_ACTION: return '../../../Dockerfile' docker_ref = ref.replace('/', '-') return f'docker://ghcr.io/{repo}:{docker_ref}' -image = set_image(EVENT, REF, REPO, REPO_ID) +image = set_image(REF, REPO, REPO_ID) action = { 'name': '🏃', From 0d02f372c36f840ef9312c8ce9d8f06d74a8a521 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Tue, 5 Nov 2024 22:29:18 +0100 Subject: [PATCH 29/44] =?UTF-8?q?=F0=9F=93=9D=F0=9F=92=85=20Update=20the?= =?UTF-8?q?=20CI/CD=20badge=20in=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a follow-up for #230, which renamed the workflow filename. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9246fd91..0ab93487 100644 --- a/README.md +++ b/README.md @@ -279,7 +279,7 @@ are released under the [BSD 3-clause license](LICENSE.md). [🧪 GitHub Actions CI/CD workflow tests badge]: -https://github.com/pypa/gh-action-pypi-publish/actions/workflows/self-smoke-test-action.yml/badge.svg?branch=unstable%2Fv1&event=push +https://github.com/pypa/gh-action-pypi-publish/actions/workflows/build-and-push-docker-image.yml/badge.svg?branch=unstable%2Fv1&event=push [GHA workflow runs list]: https://github.com/pypa/gh-action-pypi-publish/actions/workflows/self-smoke-test-action.yml?query=branch%3Aunstable%2Fv1 From dfcfeca43e4ad1597a1a8bacfca40b25ad821f61 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Nov 2024 14:00:51 +0100 Subject: [PATCH 30/44] =?UTF-8?q?=F0=9F=A7=AA=20Use=20prefetched=20action?= =?UTF-8?q?=20to=20make=20trampoline?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, the action repository was being cloned from the remote twice, unnecessarily. This patch eliminates this step and uses the copy that was checked out on job start. The generated trampoline action is still copied into the allowlisted working directory so it can be referenced by the relative path starting with `./`. It is now output under `./.github/.tmp/.generated-actions/run-pypi-publish-in-docker-container` which mutates the end-user's workspace slightly but uses a path that is unlikely to clash with somebody else's use. Unfortunately, we cannot use randomized paths because the composite action syntax does not allow accessing variables in `uses:`. Fixes #292. --- action.yml | 22 +++++++++++++--------- create-docker-action.py | 20 ++++++++++++++++++-- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/action.yml b/action.yml index 7f6b7461..f4dea680 100644 --- a/action.yml +++ b/action.yml @@ -130,24 +130,28 @@ runs: PR_REF: ${{ github.event.pull_request.head.ref }} PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} PR_REPO_ID: ${{ github.event.pull_request.base.repo.id }} - - name: Check out action repo - uses: actions/checkout@v4 - with: - path: action-repo - ref: ${{ steps.set-repo-and-ref.outputs.ref }} - repository: ${{ steps.set-repo-and-ref.outputs.repo }} - name: Create Docker container action run: | # Create Docker container action - python create-docker-action.py + python ${{ github.action_path }}/create-docker-action.py env: REF: ${{ steps.set-repo-and-ref.outputs.ref }} REPO: ${{ steps.set-repo-and-ref.outputs.repo }} REPO_ID: ${{ steps.set-repo-and-ref.outputs.repo-id }} shell: bash - working-directory: action-repo - name: Run Docker container - uses: ./action-repo/.github/actions/run-docker-container + # The generated trampoline action must exist in the allowlisted + # runner-defined working directory so it can be referenced by the + # relative path starting with `./`. + # + # This mutates the end-user's workspace slightly but uses a path + # that is unlikely to clash with somebody else's use. + # + # We cannot use randomized paths because the composite action + # syntax does not allow accessing variables in `uses:`. This + # means that we end up having to hardcode this path both here and + # in `create-docker-action.py`. + uses: ./.github/.tmp/.generated-actions/run-pypi-publish-in-docker-container with: user: ${{ inputs.user }} password: ${{ inputs.password }} diff --git a/create-docker-action.py b/create-docker-action.py index 16aa54c0..6829a9fe 100644 --- a/create-docker-action.py +++ b/create-docker-action.py @@ -10,10 +10,12 @@ REPO_ID = os.environ['REPO_ID'] REPO_ID_GH_ACTION = '178055147' +ACTION_SHELL_CHECKOUT_PATH = pathlib.Path(__file__).parent.resolve() + def set_image(ref: str, repo: str, repo_id: str) -> str: if repo_id == REPO_ID_GH_ACTION: - return '../../../Dockerfile' + return str(ACTION_SHELL_CHECKOUT_PATH / 'Dockerfile') docker_ref = ref.replace('/', '-') return f'docker://ghcr.io/{repo}:{docker_ref}' @@ -70,6 +72,20 @@ def set_image(ref: str, repo: str, repo_id: str) -> str: }, } -action_path = pathlib.Path('.github/actions/run-docker-container/action.yml') +# The generated trampoline action must exist in the allowlisted +# runner-defined working directory so it can be referenced by the +# relative path starting with `./`. +# +# This mutates the end-user's workspace slightly but uses a path +# that is unlikely to clash with somebody else's use. +# +# We cannot use randomized paths because the composite action +# syntax does not allow accessing variables in `uses:`. This +# means that we end up having to hardcode this path both here and +# in `action.yml`. +action_path = pathlib.Path( + '.github/.tmp/.generated-actions/' + 'run-pypi-publish-in-docker-container/action.yml', +) action_path.parent.mkdir(parents=True, exist_ok=True) action_path.write_text(json.dumps(action, ensure_ascii=False), encoding='utf-8') From 9b4dfb0c84502497080469ffeb5e75f9c5cce5c2 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Wed, 6 Nov 2024 16:13:48 +0100 Subject: [PATCH 31/44] =?UTF-8?q?=E2=9C=A8=20Pre-install=20Python=20if=20t?= =?UTF-8?q?here's=20none?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is not usually the case for GitHub-hosted Runners but it might happen with self-hosted runners. Fixes #289. --- action.yml | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/action.yml b/action.yml index f4dea680..3e39fd14 100644 --- a/action.yml +++ b/action.yml @@ -130,10 +130,26 @@ runs: PR_REF: ${{ github.event.pull_request.head.ref }} PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} PR_REPO_ID: ${{ github.event.pull_request.base.repo.id }} + - name: Discover pre-installed Python + id: pre-installed-python + run: | + # 🔎 Discover pre-installed Python + echo "python-path=$(command -v python3 || :)" | tee -a "${GITHUB_OUTPUT}" + shell: bash + - name: Install Python 3 + if: steps.pre-installed-python.outputs.python-path == '' + id: new-python + uses: actions/setup-python@v5 + with: + python-version: 3.x - name: Create Docker container action run: | # Create Docker container action - python ${{ github.action_path }}/create-docker-action.py + ${{ + steps.pre-installed-python.outputs.python-path == '' + && steps.new-python.outputs.python-path + || steps.pre-installed-python.outputs.python-path + }} '${{ github.action_path }}/create-docker-action.py' env: REF: ${{ steps.set-repo-and-ref.outputs.ref }} REPO: ${{ steps.set-repo-and-ref.outputs.repo }} From fec2f0c0cea943a654e4c65b547300a042e8af56 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 6 Nov 2024 13:43:44 -0500 Subject: [PATCH 32/44] attestations: collect *.zip sdists as well Signed-off-by: William Woodruff --- attestations.py | 1 + 1 file changed, 1 insertion(+) diff --git a/attestations.py b/attestations.py index d8bca495..8c66cff3 100644 --- a/attestations.py +++ b/attestations.py @@ -54,6 +54,7 @@ def debug(msg: str): def collect_dists(packages_dir: Path) -> list[Path]: # Collect all sdists and wheels. dist_paths = [sdist.resolve() for sdist in packages_dir.glob('*.tar.gz')] + dist_paths.extend(sdist.resolve() for sdist in packages_dir.glob('*.zip')) dist_paths.extend(whl.resolve() for whl in packages_dir.glob('*.whl')) # Make sure everything that looks like a dist actually is one. From fe8d1484ba76730fec68a234daa6dbe243aa2af0 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Wed, 6 Nov 2024 17:53:10 -0500 Subject: [PATCH 33/44] requirements: bump pypi-attestations to 0.0.15 Signed-off-by: William Woodruff --- requirements/runtime.in | 2 +- requirements/runtime.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index 0868f815..ee80ce1a 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -10,7 +10,7 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.13 +pypi-attestations ~= 0.0.15 sigstore ~= 3.5.1 # NOTE: Used to detect the PyPI package name from the distribution files diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 13b5f7c0..77a39596 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -93,7 +93,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.13 +pypi-attestations==0.0.15 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto From f81cd95ad93549115c42c5ad0512552fc64dde4f Mon Sep 17 00:00:00 2001 From: Brendon Smith Date: Mon, 11 Nov 2024 18:01:00 -0500 Subject: [PATCH 34/44] Tag Docker images with Git SHA PR https://github.com/pypa/gh-action-pypi-publish/pull/230 updated the action to pull Docker images from GHCR instead of building Docker images each time the workflow runs. As part of this PR, a new GitHub Actions workflow was added that builds Docker images and pushes them to GitHub Container Registry (GHCR). Actions can be referenced in various ways. The Docker build workflow covers most of the action references, but does not push Docker images tagged with the Git commit ID (Git SHA). This commit will add Docker tags for referencing the action with a Git SHA. GitHub Actions only supports references by the full 40 character SHA. If users try to reference the action by a short SHA like `1234567`, they will get an error like, "Unable to resolve action `pypa/gh-action-pypi-publish@1234567`, the provided ref `1234567` is the shortened version of a commit SHA, which is not supported. Please use the full commit SHA `1234567890123456789012345678901234567890` instead." https://github.com/pypa/gh-action-pypi-publish/pull/230 https://github.com/pypa/gh-action-pypi-publish/issues/290 https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/using-pre-written-building-blocks-in-your-workflow#using-shas --- .github/workflows/build-and-push-docker-image.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml index d8fd716c..271de1e0 100644 --- a/.github/workflows/build-and-push-docker-image.yml +++ b/.github/workflows/build-and-push-docker-image.yml @@ -32,15 +32,18 @@ jobs: IMAGE="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG}" IMAGE_MAJOR="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG_MAJOR}" IMAGE_MAJOR_MINOR="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG_MAJOR_MINOR}" + IMAGE_SHA="ghcr.io/$GITHUB_REPOSITORY:${GITHUB_SHA}" echo "IMAGE=$IMAGE" >>"$GITHUB_ENV" echo "IMAGE_MAJOR=$IMAGE_MAJOR" >>"$GITHUB_ENV" echo "IMAGE_MAJOR_MINOR=$IMAGE_MAJOR_MINOR" >>"$GITHUB_ENV" + echo "IMAGE_SHA=$IMAGE_SHA" >>"$GITHUB_ENV" docker build . \ --build-arg BUILDKIT_INLINE_CACHE=1 \ --cache-from $IMAGE \ --tag $IMAGE docker tag $IMAGE $IMAGE_MAJOR docker tag $IMAGE $IMAGE_MAJOR_MINOR + docker tag $IMAGE $IMAGE_SHA env: DOCKER_TAG: ${{ inputs.tag || github.ref_name }} - name: Log in to GHCR @@ -52,3 +55,4 @@ jobs: docker push $IMAGE docker push $IMAGE_MAJOR docker push $IMAGE_MAJOR_MINOR + docker push $IMAGE_SHA From 7c5c585c36acb0e71e5fdefcdb3429aac6dc4fe3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Fri, 22 Nov 2024 12:58:46 -0500 Subject: [PATCH 35/44] oidc-exchange: add workflow_ref to debug msg Signed-off-by: William Woodruff --- oidc-exchange.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/oidc-exchange.py b/oidc-exchange.py index 0ada77bd..227b9c4a 100644 --- a/oidc-exchange.py +++ b/oidc-exchange.py @@ -83,6 +83,7 @@ * `repository`: `{repository}` * `repository_owner`: `{repository_owner}` * `repository_owner_id`: `{repository_owner_id}` +* `workflow_ref`: `{workflow_ref}` * `job_workflow_ref`: `{job_workflow_ref}` * `ref`: `{ref}` @@ -175,6 +176,7 @@ def _get(name: str) -> str: # noqa: WPS430 repository=_get('repository'), repository_owner=_get('repository_owner'), repository_owner_id=_get('repository_owner_id'), + workflow_ref=_get('workflow_ref'), job_workflow_ref=_get('job_workflow_ref'), ref=_get('ref'), ) From 0e107253951d06521a1306887283f08498b8f4d3 Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Sun, 1 Dec 2024 12:05:46 -0500 Subject: [PATCH 36/44] requirements: bump twine to ~= 6.0 Signed-off-by: William Woodruff --- requirements/runtime.in | 2 +- requirements/runtime.txt | 7 ++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index ee80ce1a..cd43891b 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,4 +1,4 @@ -twine +twine ~= 6.0 # NOTE: Used to detect an ambient OIDC credential for OIDC publishing, # NOTE: as well as PEP 740 attestations. diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 77a39596..d50cb0a4 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -41,8 +41,6 @@ idna==3.7 # via # email-validator # requests -importlib-metadata==7.1.0 - # via twine jaraco-classes==3.4.0 # via keyring jaraco-context==5.3.0 @@ -67,6 +65,7 @@ packaging==24.1 # via # -r runtime.in # pypi-attestations + # twine pkginfo==1.10.0 # via twine platformdirs==4.2.2 @@ -133,7 +132,7 @@ six==1.16.0 # via python-dateutil tuf==5.0.0 # via sigstore -twine==5.1.1 +twine==6.0.1 # via -r runtime.in typing-extensions==4.11.0 # via @@ -143,5 +142,3 @@ urllib3==2.2.1 # via # requests # twine -zipp==3.18.2 - # via importlib-metadata From 0a0a6ae824040d7349dd2b2471a7907b86b45074 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 6 Dec 2024 04:44:41 +0100 Subject: [PATCH 37/44] =?UTF-8?q?=F0=9F=A7=AA=20Allow=20CI=20to=20register?= =?UTF-8?q?=20multiple=20distributions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is necessary to allow the smoke test check uploading multiple packages. --- .github/workflows/reusable-smoke-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/reusable-smoke-test.yml b/.github/workflows/reusable-smoke-test.yml index ac59f08f..29f52d56 100644 --- a/.github/workflows/reusable-smoke-test.yml +++ b/.github/workflows/reusable-smoke-test.yml @@ -97,7 +97,12 @@ jobs: - name: Build the stub package sdist and wheel distributions run: python3 -m build - name: Register the stub package in devpi - run: twine register dist/*.tar.gz + run: | + for dist in dist/*.tar.gz + do + echo "Registering ${dist}..." + twine register "${dist}" + done env: TWINE_USERNAME: ${{ env.devpi-username }} TWINE_PASSWORD: ${{ env.devpi-password }} From ff2b051b0afcb29a320583463b190216bbf80be4 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Fri, 6 Dec 2024 04:21:39 +0100 Subject: [PATCH 38/44] =?UTF-8?q?=F0=9F=A7=AA=20Add=20a=20Maturin-based=20?= =?UTF-8?q?package=20to=20CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/reusable-smoke-test.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/reusable-smoke-test.yml b/.github/workflows/reusable-smoke-test.yml index 29f52d56..1b59efae 100644 --- a/.github/workflows/reusable-smoke-test.yml +++ b/.github/workflows/reusable-smoke-test.yml @@ -96,6 +96,24 @@ jobs: readme = "README.md" - name: Build the stub package sdist and wheel distributions run: python3 -m build + - name: Create the Rust package directory + run: mkdir -pv rust-example + - name: Initialize a Rust project + run: cargo init + working-directory: rust-example + - name: Populate the Rust package `pyproject.toml` + run: echo "$CONTENTS" > pyproject.toml + env: + CONTENTS: | + [build-system] + requires = [ + "maturin ~=1.0", + ] + build-backend = "maturin" + working-directory: rust-example + - name: Build the stub package sdist and wheel distributions + run: python3 -m build -o ../dist/ + working-directory: rust-example - name: Register the stub package in devpi run: | for dist in dist/*.tar.gz From 138a1215a3f0562a56c666c244d8f25a8e874e5b Mon Sep 17 00:00:00 2001 From: William Woodruff Date: Thu, 5 Dec 2024 09:42:44 -0500 Subject: [PATCH 39/44] =?UTF-8?q?=F0=9F=93=8C=F0=9F=93=A6=20Pin=20`pkginfo?= =?UTF-8?q?`=20to=20v1.12=20@=20runtime=20deps?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: William Woodruff --- requirements/runtime.in | 5 +++++ requirements/runtime.txt | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/requirements/runtime.in b/requirements/runtime.in index cd43891b..63d2a7d6 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,5 +1,10 @@ twine ~= 6.0 +# NOTE: 1.12.0 and later enable support for metadata 2.4 +# NOTE: This can be dropped once twine stops using pkginfo +# Ref: https://github.com/pypa/twine/pull/1180 +pkginfo ~= 1.12.0 + # NOTE: Used to detect an ambient OIDC credential for OIDC publishing, # NOTE: as well as PEP 740 attestations. id ~= 1.0 diff --git a/requirements/runtime.txt b/requirements/runtime.txt index d50cb0a4..638b3cf0 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -66,8 +66,10 @@ packaging==24.1 # -r runtime.in # pypi-attestations # twine -pkginfo==1.10.0 - # via twine +pkginfo==1.12.0 + # via + # -r runtime.in + # twine platformdirs==4.2.2 # via sigstore pyasn1==0.6.0 From 43caae4bb174f4ce5ae7e6d8bb85eb54f0fd9e80 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 7 Dec 2024 02:24:42 +0100 Subject: [PATCH 40/44] =?UTF-8?q?=F0=9F=92=85=F0=9F=93=A6=20Split=20transi?= =?UTF-8?q?tive=20dep=20constraints?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is a structural change allowing for better placement of direct dependencies and limiting the transitive ones. --- requirements/runtime-constraints.in | 20 ++++++++++++++++++++ requirements/runtime.in | 7 ++----- requirements/runtime.txt | 2 +- 3 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 requirements/runtime-constraints.in diff --git a/requirements/runtime-constraints.in b/requirements/runtime-constraints.in new file mode 100644 index 00000000..a934c360 --- /dev/null +++ b/requirements/runtime-constraints.in @@ -0,0 +1,20 @@ +############################################################################### +# # +# This file is only meant to exclude broken dependency versions, not feature # +# dependencies. # +# # +# GUIDELINES: # +# 1. Only list PyPI project versions that need to be excluded using `!=` # +# and `<`. # +# 2. It is allowed to have transitive dependency limitations in this file. # +# 3. Apply bare minimum constraints under narrow conditions, use # +# environment markers if possible. E.g. `; python_version < "3.12"`. # +# 4. Whenever there are no constraints, let the file and this header # +# remain in Git. # +# # +############################################################################### + +# NOTE: 1.12.0 and later enable support for metadata 2.4 +# NOTE: This can be dropped once twine stops using pkginfo +# Ref: https://github.com/pypa/twine/pull/1180 +pkginfo ~= 1.12.0 diff --git a/requirements/runtime.in b/requirements/runtime.in index 63d2a7d6..c2060b79 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,9 +1,6 @@ -twine ~= 6.0 +-c runtime-constraints.in # limits known broken versions -# NOTE: 1.12.0 and later enable support for metadata 2.4 -# NOTE: This can be dropped once twine stops using pkginfo -# Ref: https://github.com/pypa/twine/pull/1180 -pkginfo ~= 1.12.0 +twine ~= 6.0 # NOTE: Used to detect an ambient OIDC credential for OIDC publishing, # NOTE: as well as PEP 740 attestations. diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 638b3cf0..98bbda9a 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -68,7 +68,7 @@ packaging==24.1 # twine pkginfo==1.12.0 # via - # -r runtime.in + # -c runtime-constraints.in # twine platformdirs==4.2.2 # via sigstore From a536fa950501c91689aa954f1d7b15c0503b6fc6 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 7 Dec 2024 02:25:27 +0100 Subject: [PATCH 41/44] =?UTF-8?q?=F0=9F=93=8C=F0=9F=93=A6=20Include=20jeep?= =?UTF-8?q?ney=20&=20secretstorage=20pins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It appears these have been missed when updating `cryptography`. This is probably dependabot's fault. --- requirements/runtime.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 98bbda9a..ac8be5d9 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -18,6 +18,7 @@ cryptography==42.0.7 # via # pyopenssl # pypi-attestations + # secretstorage # sigstore dnspython==2.6.1 # via email-validator @@ -47,6 +48,10 @@ jaraco-context==5.3.0 # via keyring jaraco-functools==4.0.1 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage keyring==25.2.1 # via twine markdown-it-py==3.0.0 @@ -118,6 +123,8 @@ rich==13.7.1 # via # sigstore # twine +secretstorage==3.3.3 + # via keyring securesystemslib==1.0.0 # via tuf sigstore==3.5.1 From 7252a9a09cc96cd5a356936f3d7570445b30bd8d Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 7 Dec 2024 05:13:12 +0100 Subject: [PATCH 42/44] =?UTF-8?q?=F0=9F=93=9D=20Outline=20unsupported=20sc?= =?UTF-8?q?enarios=20in=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/README.md b/README.md index 0ab93487..59689b1c 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ walkthrough check out the [PyPA guide]. If you have any feedback regarding specific action versions, please leave comments in the corresponding [per-release announcement discussions]. +> [!TIP] +> A limited number of usage scenarios is supported, including the +> [PyPA guide] example. See the [non-goals] for more detail. + ## 🌇 `master` branch sunset ❗ @@ -131,6 +135,9 @@ same identity. This GitHub Action [has nothing to do with _building package distributions_]. Users are responsible for preparing dists for upload by putting them into the `dist/` folder prior to running this Action. +They are typically expected to do this in a _separate GitHub Actions +CI/CD job_ running before the one where they call this action and having +restricted privileges. > [!IMPORTANT] > Since this GitHub Action is docker-based, it can only @@ -155,6 +162,72 @@ by putting them into the `dist/` folder prior to running this Action. > sharing the built dists across stages and jobs. Then, use the `needs` > setting to order the build, test and publish stages. +The expected environment for running `pypi-publish` is the +GitHub-provided Ubuntu VM. We are running a smoke-test against +`ubuntu-latest` in CI but any currently available numbered versions +should do. We'll consider them supported for as long as GitHub itself +supports them. + +Running the action in a job that has a `container:` set is not +supported. It might work for you but you're on your own when it breaks. +If you feel the need to use it, it's likely that you're not following +the recommendation of invoking the build automation in a separate job, +which is considered a security issue (especially, when using [Trusted +Publishing][trusted publisher] that may cause privilege escalation and +would enable the attackers to impersonate the GitHub-backed identity of +the repository through transitive build dependency poisoning). The +solution is to have one job (or multiple, in case of projects with +C-extensions) for building the distribution packages, followed by +another that publishes them. + +Self-hosted runners are best effort, provided no other unsupported +things influence them. We are unable to test this in CI and they may +break. This is often the case when using custom runtimes and not the +official GitHub-provided VMs. In general, if you follow the +recommendation of building in a separate job, you shouldn't need to run +this action within a self-hosted runner — it should be possible to +build your dists in a self-hosted runner, save them as a GitHub Actions +artifact in that job, and then invoke the publishing job that would run +within GitHub-provided runners, downloading the artifact with the dists +and publishing them. Such separation is the _recommended_/**supported** +way of handling this scenario. +Our understandng is that Trusted publishing is expected to work on +self-hosted runners. It is backed by OIDC. If it doesn't work, you +should probably ask GitHub if you missed something. We wouldn't be able +to assist here. + +Trusted Publishing cannot be tested in CI at the moment, sadly. It is +supported and bugs should be reported but it may take time to sort out +as it often requires cross-project collaboration to debug (sometimes, +problems occur due to changes in PyPI and not in the action). + +The only case that is explicitly unsupported at the moment is [Trusted +Publishing][trusted publisher] in reusable workflows. This requires +support on the PyPI side and is being worked on. Please, do not report +bugs related to this case. The current recommendation is to put +everything else you want into a reusable workflow but keep the job +calling `pypi-publish` in a top-level one. + +Invoking `pypi-publish` from composite actions is unsupported. It is not +tested. GitHub Runners have limitations and bugs in this case. But more +importantly, this is usually an indication of using it insecurely. When +using [Trusted Publishing][trusted publisher], it is imperative to keep +build machinery invocation in a separate job with restrictive priviliges +as [Trusted Publishing][trusted publisher] itself requires elevated +permissions to make use of OIDC. Our observation is that the users +sometimes create in-project composite actions that invoke building and +publishing in the same job. As such, we don't seek to support such a +dangerous configuration in the first place. The solution is pretty much +the same as with the previous problem — use a separate job with +dedicated and scoped privileges just for publishing; and invoke that +in-project composite action from a different job. + +And finally, invoking `pypi-publish` more than once in the same job is +not considered supported. It may work in a limited number of scenarios +but please, don't do this. If you want to publish to several indexes, +build the dists in one job and add several publishing jobs, one per +upload. + ## Advanced release management @@ -294,6 +367,8 @@ https://julienrenaux.fr/2019/12/20/github-actions-security-risk/ [per-release announcement discussions]: https://github.com/pypa/gh-action-pypi-publish/discussions/categories/announcements +[non-goals]: #Non-goals + [Creating & using secrets]: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets [has nothing to do with _building package distributions_]: From cbd6d01d855e02aab0908c7709d5c0ddc88c617a Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Sat, 7 Dec 2024 05:17:14 +0100 Subject: [PATCH 43/44] =?UTF-8?q?=F0=9F=93=9DFix=20a=20typo=20in=20"privil?= =?UTF-8?q?eges"=20@=20README?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 59689b1c..8ddf0d7d 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ Invoking `pypi-publish` from composite actions is unsupported. It is not tested. GitHub Runners have limitations and bugs in this case. But more importantly, this is usually an indication of using it insecurely. When using [Trusted Publishing][trusted publisher], it is imperative to keep -build machinery invocation in a separate job with restrictive priviliges +build machinery invocation in a separate job with restrictive privileges as [Trusted Publishing][trusted publisher] itself requires elevated permissions to make use of OIDC. Our observation is that the users sometimes create in-project composite actions that invoke building and From 67339c736fd9354cd4f8cb0b744f2b82a74b5c70 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Mon, 9 Dec 2024 15:07:39 +0100 Subject: [PATCH 44/44] =?UTF-8?q?=F0=9F=93=A6=20Only=20keep=20lower=20boun?= =?UTF-8?q?ds=20@=20input=20requirements?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This concerns both direct (`twine`) and indirect (`pkginfo`) deps, provided there's no broken versions to exclude. --- requirements/runtime-constraints.in | 2 +- requirements/runtime.in | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements/runtime-constraints.in b/requirements/runtime-constraints.in index a934c360..e4afedd0 100644 --- a/requirements/runtime-constraints.in +++ b/requirements/runtime-constraints.in @@ -17,4 +17,4 @@ # NOTE: 1.12.0 and later enable support for metadata 2.4 # NOTE: This can be dropped once twine stops using pkginfo # Ref: https://github.com/pypa/twine/pull/1180 -pkginfo ~= 1.12.0 +pkginfo >= 1.12.0 diff --git a/requirements/runtime.in b/requirements/runtime.in index c2060b79..5861e0b7 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,6 +1,7 @@ -c runtime-constraints.in # limits known broken versions -twine ~= 6.0 +# NOTE: v6 is needed to support metadata v2.4 +twine >= 6.0 # NOTE: Used to detect an ambient OIDC credential for OIDC publishing, # NOTE: as well as PEP 740 attestations.