Add a comment trigger for rerunning failed Buildkite jobs (#42655)

Author: DilumAluthge

.github/workflows/rerun_failed.yml

@@ -0,0 +1,87 @@
+# Please ping @DilumAluthge when making any changes to this file.
+
+# Here are some steps that we take in this workflow file for security reasons:
+# 1. We do not checkout any code.
+# 2. We only run actions that are defined in a repository in the `JuliaLang` GitHub organization.
+# 3. We do not give the `GITHUB_TOKEN` any permissions.
+# 4. We only give the Buildkite API token (`BUILDKITE_API_TOKEN_RETRY`) the minimum necessary
+#    set of permissions.
+
+# Important note to Buildkite maintainers:
+# In order to make this work, you need to tell Buildkite that it should NOT create a brand-new
+# build when someone closes and reopens a pull request. To do so:
+# 1. Go to the relevant pipeline (e.g. https://buildkite.com/julialang/julia-master).
+# 2. Click on the "Pipeline Settings" button.
+# 3. In the left sidebar, under "Pipeline Settings", click on "GitHub".
+# 4. In the "GitHub Settings", under "Build Pull Requests", make sure that the "Skip pull
+#    request builds for existing commits" checkbox is checked. This is the setting that tells
+#    Buildkite that it should NOT create a brand-new build when someone closes and reopens a
+#    pull request.
+# 5. At the bottom of the page, click the "Save GitHub Settings" button.
+
+name: Rerun Failed Buildkite Jobs
+
+# There are two ways that a user can rerun the failed Buildkite jobs:
+# 1. Close and reopen the pull request.
+#    In order to use this approach, the user must be in one of the following three categories:
+#        (i)   Author of the pull request
+#        (ii)  Commit permissions
+#        (iii) Triage permissions
+# 2. Post a comment on the pull request with exactly the following contents: /buildkite rerun failed
+#    In order to use this approach, the user must be in the following category:
+#        - A member of the JuliaLang GitHub organization (the membership must be publicized)
+
+on:
+  # When using the `pull_request_target` event, all PRs will get access to secret environment
+  # variables (such as the `BUILDKITE_API_TOKEN_RETRY` secret environment variable), even if
+  # the PR is from a fork. Therefore, for security reasons, we do not checkout any code in
+  # this workflow.
+  pull_request_target:
+    types: [ reopened ]
+  issue_comment:
+    types: [ created ]
+
+# We do not give the `GITHUB_TOKEN` any permissions.
+# Therefore, the `GITHUB_TOKEN` only has the same access as any member of the public.
+permissions:
+  contents: none
+
+jobs:
+  rerun-failed-buildkite-jobs:
+    name: Rerun Failed Buildkite Jobs
+    runs-on: ubuntu-latest
+    if: (github.repository == 'DilumAluthge/julia') && ((github.event_name == 'pull_request_target' && github.event.action == 'reopened') || (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == '/buildkite rerun failed'))
+    steps:
+      - name: Check organization membership
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then
+            if [[ "${{ github.event.action }}" == "reopened" ]]; then
+              echo "This is a \"reopened\" event, so we do not need to check the user's organization membership."
+              echo "GOOD_TO_PROCEED=yes" >> ${GITHUB_ENV:?}
+            else
+              echo "ERROR: The github.event_name is \"pull_request_target\", but the github.event.action is not \"reopened\"."
+              exit 1
+            fi
+          else
+            curl -H "Authorization: token ${GITHUB_TOKEN:?}" "https://api.github.com/users/${{ github.event.sender.login }}"
+            curl -H "Authorization: token ${GITHUB_TOKEN:?}" "https://api.github.com/users/${{ github.event.sender.login }}/orgs"
+            export USER_IS_ORGANIZATION_MEMBER=`curl -H "Authorization: token ${GITHUB_TOKEN:?}" "https://api.github.com/users/${{ github.event.sender.login }}/orgs" | jq '[.[] | .login] | index("JuliaLang") != null' | tr -s ' '`
+            if [[ "${USER_IS_ORGANIZATION_MEMBER:?}"   == "true" ]]; then
+              echo "The \"${{ github.event.sender.login }}\" user is a public member of the JuliaLang organization."
+              echo "GOOD_TO_PROCEED=yes" >> ${GITHUB_ENV:?}
+            else
+              echo "ERROR: the \"${{ github.event.sender.login }}\" user is NOT a public member of the JuliaLang organization."
+              echo "If you are a member, please make sure that you have publicized your membership."
+              exit 1
+            fi
+          fi
+      # For security reasons, we do not checkout any code in this workflow.
+      - uses: JuliaLang/buildkite-rerun-failed@057f6f2d37aa29a57b7679fd2af0df1d9f9188b4
+        if: env.GOOD_TO_PROCEED == 'yes'
+        with:
+          buildkite_api_token: ${{ secrets.BUILDKITE_API_TOKEN_RETRY }}
+          buildkite_organization_slug: 'julialang'
+          buildkite_pipeline_slug: 'julia-master'
+          pr_number: ${{github.event.number}}