From 85f1370606392755b38ebd6a2f8e1dc978d41d9d Mon Sep 17 00:00:00 2001 From: Manish Goregaokar Date: Fri, 17 Jul 2020 18:13:42 -0700 Subject: [PATCH] Rollup merge of #74418 - rye:gha-dedup-shell-setting, r=pietroalbini ci: Set `shell: bash` as a default, remove duplicates A follow-up to #74406, this commit merely removes the `shell: bash` lines where they were added in favor of setting defaults for *all* "run" steps in the jobs that run the tests. The changes in #74406 were needed because of an upstream change to the `windows-2019` GitHub Actions image. Previously, the configuration worked fine without specifying `shell: bash`, but for some reason this broke with a new change that was deployed today. The preceding PR was a hotfix to get CI passing, but there was a slightly less duplicative way to specify the default shell for the jobs, which was to set the `defaults.run` option. This change applies to the `pr`, `try`, `auto`, and `auto-fallible` jobs, which are derived from the YAML-anchor `base-ci-job`. I did not apply these changes to the `master`, `try-success`, `try-failure`, `auto-success`, or `auto-failure` jobs because they have only a few steps. cc/r? @Mark-Simulacrum --- .github/workflows/ci.yml | 82 +-------------------------------- src/ci/exec-with-shell.py | 16 ------- src/ci/github-actions/ci.yml | 36 ++------------- src/ci/scripts/install-msys2.sh | 3 -- 4 files changed, 5 insertions(+), 132 deletions(-) delete mode 100755 src/ci/exec-with-shell.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 85ec3307518c5..c1a2505883106 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ name: CI - "**" defaults: run: - shell: "python src/ci/exec-with-shell.py {0}" + shell: bash jobs: pr: name: PR @@ -54,7 +54,6 @@ jobs: steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -66,77 +65,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -144,7 +125,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" @@ -173,7 +153,6 @@ jobs: steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -185,77 +164,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -263,7 +224,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" @@ -504,7 +464,6 @@ jobs: steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -516,77 +475,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -594,7 +535,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" @@ -649,7 +589,6 @@ jobs: steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 with: @@ -661,77 +600,59 @@ jobs: if: "success() && !env.SKIP_JOB && github.ref != 'refs/heads/try'" - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: EXTRA_VARIABLES: "${{ toJson(matrix.env) }}" if: success() && !env.SKIP_JOB - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash if: success() && !env.SKIP_JOB - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash if: success() && !env.SKIP_JOB - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash if: success() && !env.SKIP_JOB - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash if: success() && !env.SKIP_JOB - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash if: success() && !env.SKIP_JOB - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash if: success() && !env.SKIP_JOB - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash if: success() && !env.SKIP_JOB - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash if: success() && !env.SKIP_JOB - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash if: success() && !env.SKIP_JOB - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash if: success() && !env.SKIP_JOB - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash if: success() && !env.SKIP_JOB - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash if: success() && !env.SKIP_JOB - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash if: success() && !env.SKIP_JOB - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}" @@ -739,7 +660,6 @@ jobs: if: success() && !env.SKIP_JOB - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}" AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}" diff --git a/src/ci/exec-with-shell.py b/src/ci/exec-with-shell.py deleted file mode 100755 index 26ce69e33d9c3..0000000000000 --- a/src/ci/exec-with-shell.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -# A simple wrapper that forwards the arguments to bash, unless the -# CI_OVERRIDE_SHELL environment variable is present: in that case the content -# of that environment variable is used as the shell path. - -import os -import sys -import subprocess - -try: - shell = os.environ["CI_OVERRIDE_SHELL"] -except KeyError: - shell = "bash" - -res = subprocess.call([shell] + sys.argv[1:]) -sys.exit(res) diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index be0fe8c5b1762..978a0c56eb672 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -79,7 +79,6 @@ x--expand-yaml-anchors--remove: steps: - name: disable git crlf conversion run: git config --global core.autocrlf false - shell: bash - name: checkout the source code uses: actions/checkout@v1 @@ -95,7 +94,6 @@ x--expand-yaml-anchors--remove: - name: add extra environment variables run: src/ci/scripts/setup-environment.sh - shell: bash env: # Since it's not possible to merge `${{ matrix.env }}` with the other # variables in `job..env`, the variables defined in the matrix @@ -106,67 +104,54 @@ x--expand-yaml-anchors--remove: - name: decide whether to skip this job run: src/ci/scripts/should-skip-this.sh - shell: bash <<: *step - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh - shell: bash <<: *step - name: show the current environment run: src/ci/scripts/dump-environment.sh - shell: bash <<: *step - name: install awscli run: src/ci/scripts/install-awscli.sh - shell: bash <<: *step - name: install sccache run: src/ci/scripts/install-sccache.sh - shell: bash <<: *step - name: install clang run: src/ci/scripts/install-clang.sh - shell: bash <<: *step - name: install WIX run: src/ci/scripts/install-wix.sh - shell: bash <<: *step - name: ensure the build happens on a partition with enough space run: src/ci/scripts/symlink-build-dir.sh - shell: bash <<: *step - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash <<: *step - name: install MSYS2 run: src/ci/scripts/install-msys2.sh - shell: bash <<: *step - name: install MinGW run: src/ci/scripts/install-mingw.sh - shell: bash <<: *step - name: install ninja run: src/ci/scripts/install-ninja.sh - shell: bash <<: *step - name: enable ipv6 on Docker run: src/ci/scripts/enable-docker-ipv6.sh - shell: bash <<: *step # Disable automatic line ending conversion (again). On Windows, when we're @@ -176,22 +161,18 @@ x--expand-yaml-anchors--remove: # appropriate line endings. - name: disable git crlf conversion run: src/ci/scripts/disable-git-crlf-conversion.sh - shell: bash <<: *step - name: checkout submodules run: src/ci/scripts/checkout-submodules.sh - shell: bash <<: *step - name: ensure line endings are correct run: src/ci/scripts/verify-line-endings.sh - shell: bash <<: *step - name: run the build run: src/ci/scripts/run-build-from-ci.sh - shell: bash env: AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }} @@ -200,7 +181,6 @@ x--expand-yaml-anchors--remove: - name: upload artifacts to S3 run: src/ci/scripts/upload-artifacts.sh - shell: bash env: AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }} @@ -250,18 +230,10 @@ on: defaults: run: - # While on Linux and macOS builders it just forwards the arguments to the - # system bash, this wrapper allows switching from the host's bash.exe to - # the one we install along with MSYS2 mid-build on Windows. - # - # Once the step to install MSYS2 is executed, the CI_OVERRIDE_SHELL - # environment variable is set pointing to our MSYS2's bash.exe. From that - # moment the host's bash.exe will not be called anymore. - # - # This is needed because we can't launch our own bash.exe from the host - # bash.exe, as that would load two different cygwin1.dll in memory, causing - # "cygwin heap mismatch" errors. - shell: python src/ci/exec-with-shell.py {0} + # On Linux, macOS, and Windows, use the system-provided bash as the default + # shell. (This should only make a difference on Windows, where the default + # shell is PowerShell.) + shell: bash jobs: pr: diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh index 3a0c965a67710..185d361582505 100755 --- a/src/ci/scripts/install-msys2.sh +++ b/src/ci/scripts/install-msys2.sh @@ -12,9 +12,6 @@ if isWindows; then mkdir -p "${msys2Path}/home/${USERNAME}" ciCommandAddPath "${msys2Path}/usr/bin" - echo "switching shell to use our own bash" - ciCommandSetEnv CI_OVERRIDE_SHELL "${msys2Path}/usr/bin/bash.exe" - # Detect the native Python version installed on the agent. On GitHub # Actions, the C:\hostedtoolcache\windows\Python directory contains a # subdirectory for each installed Python version.