diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml index 4442afc98e406..24b07a1b7c950 100644 --- a/src/ci/azure-pipelines/auto.yml +++ b/src/ci/azure-pipelines/auto.yml @@ -315,6 +315,7 @@ jobs: # 32/64 bit MSVC and GNU deployment dist-x86_64-msvc: + MSYS_BITS: 64 RUST_CONFIGURE_ARGS: > --build=x86_64-pc-windows-msvc --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc @@ -324,6 +325,7 @@ jobs: DIST_REQUIRE_ALL_TOOLS: 1 DEPLOY: 1 dist-i686-msvc: + MSYS_BITS: 32 RUST_CONFIGURE_ARGS: > --build=i686-pc-windows-msvc --target=i586-pc-windows-msvc diff --git a/src/ci/azure-pipelines/steps/install-clang.yml b/src/ci/azure-pipelines/steps/install-clang.yml deleted file mode 100644 index 14daf81b43075..0000000000000 --- a/src/ci/azure-pipelines/steps/install-clang.yml +++ /dev/null @@ -1,46 +0,0 @@ -steps: - -- bash: | - set -e - curl -f http://releases.llvm.org/7.0.0/clang+llvm-7.0.0-x86_64-apple-darwin.tar.xz | tar xJf - - - export CC=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang - echo "##vso[task.setvariable variable=CC]$CC" - - export CXX=`pwd`/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++ - echo "##vso[task.setvariable variable=CXX]$CXX" - - # Configure `AR` specifically so rustbuild doesn't try to infer it as - # `clang-ar` by accident. - echo "##vso[task.setvariable variable=AR]ar" - displayName: Install clang (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - -# If we're compiling for MSVC then we, like most other distribution builders, -# switch to clang as the compiler. This'll allow us eventually to enable LTO -# amongst LLVM and rustc. Note that we only do this on MSVC as I don't think -# clang has an output mode compatible with MinGW that we need. If it does we -# should switch to clang for MinGW as well! -# -# Note that the LLVM installer is an NSIS installer -# -# Original downloaded here came from -# http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe -# That installer was run through `wine` on Linux and then the resulting -# installation directory (found in `$HOME/.wine/drive_c/Program Files/LLVM`) was -# packaged up into a tarball. We've had issues otherwise that the installer will -# randomly hang, provide not a lot of useful information, pollute global state, -# etc. In general the tarball is just more confined and easier to deal with when -# working with various CI environments. -- bash: | - set -e - mkdir -p citools - cd citools - curl -f https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/LLVM-7.0.0-win64.tar.gz | tar xzf - - echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --set llvm.clang-cl=`pwd`/clang-rust/bin/clang-cl.exe" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) - displayName: Install clang (Windows) - -# Note that we don't install clang on Linux since its compiler story is just so -# different. Each container has its own toolchain configured appropriately -# already. diff --git a/src/ci/azure-pipelines/steps/install-sccache.yml b/src/ci/azure-pipelines/steps/install-sccache.yml deleted file mode 100644 index d4679c1c6733e..0000000000000 --- a/src/ci/azure-pipelines/steps/install-sccache.yml +++ /dev/null @@ -1,21 +0,0 @@ -steps: - -- bash: | - set -e - curl -fo /usr/local/bin/sccache https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-02-sccache-x86_64-apple-darwin - chmod +x /usr/local/bin/sccache - displayName: Install sccache (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) - -- script: | - md sccache - powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf sccache\sccache.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2018-04-26-sccache-x86_64-pc-windows-msvc" - echo ##vso[task.prependpath]%CD%\sccache - displayName: Install sccache (Windows) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# Note that we don't install sccache on Linux since it's installed elsewhere -# through all the containers. -# -# FIXME: we should probably install sccache outside the containers and then -# mount it inside the containers so we can centralize all installation here. diff --git a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml b/src/ci/azure-pipelines/steps/install-windows-build-deps.yml deleted file mode 100644 index 812339900fe42..0000000000000 --- a/src/ci/azure-pipelines/steps/install-windows-build-deps.yml +++ /dev/null @@ -1,143 +0,0 @@ -steps: -# We use the WIX toolset to create combined installers for Windows, and these -# binaries are downloaded from -# https://github.com/wixtoolset/wix3 originally -- bash: | - set -e - curl -O https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/wix311-binaries.zip - echo "##vso[task.setvariable variable=WIX]`pwd`/wix" - mkdir -p wix/bin - cd wix/bin - 7z x ../../wix311-binaries.zip - displayName: Install wix - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# We use InnoSetup and its `iscc` program to also create combined installers. -# Honestly at this point WIX above and `iscc` are just holdovers from -# oh-so-long-ago and are required for creating installers on Windows. I think -# one is MSI installers and one is EXE, but they're not used so frequently at -# this point anyway so perhaps it's a wash! -- script: | - echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5 - curl.exe -o is-install.exe https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-08-22-is.exe - is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP- - displayName: Install InnoSetup - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# We've had issues with the default drive in use running out of space during a -# build, and it looks like the `C:` drive has more space than the default `D:` -# drive. We should probably confirm this with the azure pipelines team at some -# point, but this seems to fix our "disk space full" problems. -- script: | - mkdir c:\MORE_SPACE - mklink /J build c:\MORE_SPACE - displayName: "Ensure build happens on C:/ instead of D:/" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- bash: git config --replace-all --global core.autocrlf false - displayName: "Disable git automatic line ending conversion (on C:/)" - -# Download and install MSYS2, needed primarily for the test suite (run-make) but -# also used by the MinGW toolchain for assembling things. -# -# FIXME: we should probe the default azure image and see if we can use the MSYS2 -# toolchain there. (if there's even one there). For now though this gets the job -# done. -- bash: | - set -e - choco install msys2 --params="/InstallDir:$(System.Workfolder)/msys2 /NoPath" -y --no-progress - echo "##vso[task.prependpath]$(System.Workfolder)/msys2/usr/bin" - mkdir -p "$(System.Workfolder)/msys2/home/$USERNAME" - displayName: Install msys2 - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -- bash: pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar - displayName: Install msys2 base deps - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# If we need to download a custom MinGW, do so here and set the path -# appropriately. -# -# Here we also do a pretty heinous thing which is to mangle the MinGW -# installation we just downloaded. Currently, as of this writing, we're using -# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it -# appears to be the first version which contains a fix for #40546, builds -# randomly failing during LLVM due to ar.exe/ranlib.exe failures. -# -# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds -# to contain a regression in gdb (#40184). As a result if we were to use the -# gdb provided (7.11.1) then we would fail all debuginfo tests. -# -# In order to fix spurious failures (pretty high priority) we use 6.3.0. To -# avoid disabling gdb tests we download an *old* version of gdb, specifically -# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb -# with the 6.2.0 gdb to get tests passing. -# -# Note that we don't literally overwrite the gdb.exe binary because it appears -# to just use gdborig.exe, so that's the binary we deal with instead. -- bash: | - set -e - curl -o mingw.7z $MINGW_URL/$MINGW_ARCHIVE - 7z x -y mingw.7z > /dev/null - curl -o $MINGW_DIR/bin/gdborig.exe $MINGW_URL/2017-04-20-${MSYS_BITS}bit-gdborig.exe - echo "##vso[task.prependpath]`pwd`/$MINGW_DIR/bin" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), ne(variables['MINGW_URL'],'')) - displayName: Download custom MinGW - -# FIXME(#65767): workaround msys bug, step 1 -- bash: | - set -e - arch=i686 - if [ "$MSYS_BITS" = "64" ]; then - arch=x86_64 - fi - curl -O https://ci-mirrors.rust-lang.org/rustc/msys2-repo/mingw/$arch/mingw-w64-$arch-ca-certificates-20180409-1-any.pkg.tar.xz - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Download working ca-certificates for msys - -# Otherwise install MinGW through `pacman` -- bash: | - set -e - arch=i686 - if [ "$MSYS_BITS" = "64" ]; then - arch=x86_64 - fi - pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake mingw-w64-$arch-gcc mingw-w64-$arch-python2 - echo "##vso[task.prependpath]$(System.Workfolder)/msys2/mingw$MSYS_BITS/bin" - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'), eq(variables['MINGW_URL'],'')) - displayName: Download standard MinGW - -# FIXME(#65767): workaround msys bug, step 2 -- bash: | - set -e - arch=i686 - if [ "$MSYS_BITS" = "64" ]; then - arch=x86_64 - fi - pacman -U --noconfirm --noprogressbar mingw-w64-$arch-ca-certificates-20180409-1-any.pkg.tar.xz - rm mingw-w64-$arch-ca-certificates-20180409-1-any.pkg.tar.xz - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Install working ca-certificates for msys - -# Make sure we use the native python interpreter instead of some msys equivalent -# one way or another. The msys interpreters seem to have weird path conversions -# baked in which break LLVM's build system one way or another, so let's use the -# native version which keeps everything as native as possible. -- bash: | - set -e - cp C:/Python27amd64/python.exe C:/Python27amd64/python2.7.exe - echo "##vso[task.prependpath]C:/Python27amd64" - displayName: Prefer the "native" Python as LLVM has trouble building with MSYS sometimes - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - -# Note that this is originally from the github releases patch of Ninja -- bash: | - set -e - mkdir ninja - curl -o ninja.zip https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2017-03-15-ninja-win.zip - 7z x -oninja ninja.zip - rm ninja.zip - echo "##vso[task.setvariable variable=RUST_CONFIGURE_ARGS]$RUST_CONFIGURE_ARGS --enable-ninja" - echo "##vso[task.prependpath]`pwd`/ninja" - displayName: Download and install ninja - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) diff --git a/src/ci/azure-pipelines/steps/run.yml b/src/ci/azure-pipelines/steps/run.yml index 15a2499e4609e..01858cca50b1c 100644 --- a/src/ci/azure-pipelines/steps/run.yml +++ b/src/ci/azure-pipelines/steps/run.yml @@ -48,86 +48,106 @@ steps: - bash: python src/ci/cpu-usage-over-time.py &> cpu-usage.csv & displayName: "Collect CPU-usage statistics in the background" -- bash: printenv | sort - displayName: Show environment variables +- bash: src/ci/scripts/dump-environment.sh + displayName: Show the current environment -- bash: | - set -e - df -h - du . | sort -nr | head -n100 - displayName: Show disk usage - # FIXME: this hasn't been tested, but maybe it works on Windows? Should test! - condition: and(succeeded(), ne(variables['Agent.OS'], 'Windows_NT')) - -- template: install-sccache.yml -- template: install-clang.yml - -# Switch to XCode 9.3 on OSX since it seems to be the last version that supports -# i686-apple-darwin. We'll eventually want to upgrade this and it will probably -# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. -- bash: | - set -e - sudo xcode-select --switch /Applications/Xcode_9.3.app - displayName: Switch to Xcode 9.3 (OSX) - condition: and(succeeded(), eq(variables['Agent.OS'], 'Darwin')) +- bash: src/ci/scripts/install-sccache.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install sccache + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-clang.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install clang + condition: and(succeeded(), not(variables.SKIP_JOB)) -- template: install-windows-build-deps.yml +- bash: src/ci/scripts/switch-xcode.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Switch to Xcode 9.3 + condition: and(succeeded(), not(variables.SKIP_JOB)) -# Looks like docker containers have IPv6 disabled by default, so let's turn it -# on since libstd tests require it -- bash: | - set -e - sudo mkdir -p /etc/docker - echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' | sudo tee /etc/docker/daemon.json - sudo service docker restart - displayName: Enable IPv6 - condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Linux')) +- bash: src/ci/scripts/install-wix.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install wix + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-innosetup.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install InnoSetup + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/windows-symlink-build-dir.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Ensure the build happens on C:\ instead of D:\ + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/disable-git-crlf-conversion.sh + displayName: "Disable git automatic line ending conversion (on C:/)" + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-msys2.sh + env: + AGENT_OS: $(Agent.OS) + SYSTEM_WORKFOLDER: $(System.Workfolder) + displayName: Install msys2 + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-msys2-packages.sh + env: + AGENT_OS: $(Agent.OS) + SYSTEM_WORKFOLDER: $(System.Workfolder) + displayName: Install msys2 packages + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-mingw.sh + env: + AGENT_OS: $(Agent.OS) + SYSTEM_WORKFOLDER: $(System.Workfolder) + displayName: Install MinGW + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/install-ninja.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Install ninja + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/enable-docker-ipv6.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Enable IPv6 on Docker + condition: and(succeeded(), not(variables.SKIP_JOB)) # Disable automatic line ending conversion (again). On Windows, when we're # installing dependencies, something switches the git configuration directory or # re-enables autocrlf. We've not tracked down the exact cause -- and there may # be multiple -- but this should ensure submodules are checked out with the # appropriate line endings. -- bash: git config --replace-all --global core.autocrlf false - displayName: "Disable git automatic line ending conversion" +- bash: src/ci/scripts/disable-git-crlf-conversion.sh + displayName: Disable git automatic line ending conversion + condition: and(succeeded(), not(variables.SKIP_JOB)) -# Check out all our submodules, but more quickly than using git by using one of -# our custom scripts -- bash: | - set -e - mkdir -p $HOME/rustsrc - $BUILD_SOURCESDIRECTORY/src/ci/init_repo.sh . $HOME/rustsrc - condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Windows_NT')) - displayName: Check out submodules (Unix) -- script: | - if not exist C:\cache\rustsrc\NUL mkdir C:\cache\rustsrc - sh src/ci/init_repo.sh . /c/cache/rustsrc - condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Check out submodules (Windows) - -# See also the disable for autocrlf above, this just checks that it worked -# -# We check both in rust-lang/rust and in a submodule to make sure both are -# accurate. Submodules are checked out significantly later than the main -# repository in this script, so settings can (and do!) change between then. -# -# Linux (and maybe macOS) builders don't currently have dos2unix so just only -# run this step on Windows. -- bash: | - set -x - # print out the git configuration so we can better investigate failures in - # the following - git config --list --show-origin - dos2unix -ih Cargo.lock src/tools/rust-installer/install-template.sh - endings=$(dos2unix -ic Cargo.lock src/tools/rust-installer/install-template.sh) - # if endings has non-zero length, error out - if [ -n "$endings" ]; then exit 1 ; fi - condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT')) - displayName: Verify line endings are LF +- bash: src/ci/scripts/checkout-submodules.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Checkout submodules + condition: and(succeeded(), not(variables.SKIP_JOB)) + +- bash: src/ci/scripts/verify-line-endings.sh + env: + AGENT_OS: $(Agent.OS) + displayName: Verify line endings + condition: and(succeeded(), not(variables.SKIP_JOB)) # Ensure the `aws` CLI is installed so we can deploy later on, cache docker # images, etc. -- bash: src/ci/install-awscli.sh +- bash: src/ci/scripts/install-awscli.sh env: AGENT_OS: $(Agent.OS) condition: and(succeeded(), not(variables.SKIP_JOB)) diff --git a/src/ci/run.sh b/src/ci/run.sh index 0d5ea371245e4..bce35670c8d46 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -117,7 +117,7 @@ make check-bootstrap # Display the CPU and memory information. This helps us know why the CI timing # is fluctuating. -if isOSX; then +if isMacOS; then system_profiler SPHardwareDataType || true sysctl hw || true ncpus=$(sysctl -n hw.ncpu) diff --git a/src/ci/scripts/checkout-submodules.sh b/src/ci/scripts/checkout-submodules.sh new file mode 100755 index 0000000000000..0b44ea3c90bc9 --- /dev/null +++ b/src/ci/scripts/checkout-submodules.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Check out all our submodules, but more quickly than using git by using one of +# our custom scripts + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + path="/c/cache/rustsrc" +else + path="${HOME}/rustsrc" +fi + +mkdir -p "${path}" +"$(cd "$(dirname "$0")" && pwd)/../init_repo.sh" . "${path}" diff --git a/src/ci/scripts/disable-git-crlf-conversion.sh b/src/ci/scripts/disable-git-crlf-conversion.sh new file mode 100755 index 0000000000000..836145fbb8e60 --- /dev/null +++ b/src/ci/scripts/disable-git-crlf-conversion.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Disable automatic line ending conversion, which is enabled by default on +# Azure's Windows image. Having the conversion enabled caused regressions both +# in our test suite (it broke miri tests) and in the ecosystem, since we +# started shipping install scripts with CRLF endings instead of the old LF. +# +# Note that we do this a couple times during the build as the PATH and current +# user/directory change, e.g. when mingw is enabled. + +set -euo pipefail +IFS=$'\n\t' + +git config --replace-all --global core.autocrlf false diff --git a/src/ci/scripts/dump-environment.sh b/src/ci/scripts/dump-environment.sh new file mode 100755 index 0000000000000..c6774b52ab92d --- /dev/null +++ b/src/ci/scripts/dump-environment.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# This script dumps information about the build environment to stdout. + +set -euo pipefail +IFS=$'\n\t' + +echo "environment variables:" +printenv | sort +echo + +echo "disk usage:" +df -h +echo + +echo "biggest files in the working dir:" +set +o pipefail +du . | sort -nr | head -n100 +set -o pipefail +echo diff --git a/src/ci/scripts/enable-docker-ipv6.sh b/src/ci/scripts/enable-docker-ipv6.sh new file mode 100755 index 0000000000000..03d5a75e24e27 --- /dev/null +++ b/src/ci/scripts/enable-docker-ipv6.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Looks like docker containers have IPv6 disabled by default, so let's turn it +# on since libstd tests require it + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isLinux; then + sudo mkdir -p /etc/docker + echo '{"ipv6":true,"fixed-cidr-v6":"fd9a:8454:6789:13f7::/64"}' \ + | sudo tee /etc/docker/daemon.json + sudo service docker restart +fi diff --git a/src/ci/install-awscli.sh b/src/ci/scripts/install-awscli.sh similarity index 88% rename from src/ci/install-awscli.sh rename to src/ci/scripts/install-awscli.sh index 69c8d2e3099ab..e21187938504c 100755 --- a/src/ci/install-awscli.sh +++ b/src/ci/scripts/install-awscli.sh @@ -16,12 +16,14 @@ set -euo pipefail IFS=$'\n\t' -MIRROR="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc/2019-07-27-awscli.tar" +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +MIRROR="${MIRRORS_BASE}/2019-07-27-awscli.tar" DEPS_DIR="/tmp/awscli-deps" pip="pip" pipflags="" -if [[ "${AGENT_OS}" == "Linux" ]]; then +if isLinux; then pip="pip3" pipflags="--user" diff --git a/src/ci/scripts/install-clang.sh b/src/ci/scripts/install-clang.sh new file mode 100755 index 0000000000000..e9b685718e61f --- /dev/null +++ b/src/ci/scripts/install-clang.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# This script installs clang on the local machine. Note that we don't install +# clang on Linux since its compiler story is just so different. Each container +# has its own toolchain configured appropriately already. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isMacOS; then + curl -f "${MIRRORS_BASE}/clang%2Bllvm-7.0.0-x86_64-apple-darwin.tar.xz" | tar xJf - + + ciCommandSetEnv CC "$(pwd)/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang" + ciCommandSetEnv CXX "$(pwd)/clang+llvm-7.0.0-x86_64-apple-darwin/bin/clang++" + + # Configure `AR` specifically so rustbuild doesn't try to infer it as + # `clang-ar` by accident. + ciCommandSetEnv AR "ar" +elif isWindows && [[ -z ${MINGW_URL+x} ]]; then + # If we're compiling for MSVC then we, like most other distribution builders, + # switch to clang as the compiler. This'll allow us eventually to enable LTO + # amongst LLVM and rustc. Note that we only do this on MSVC as I don't think + # clang has an output mode compatible with MinGW that we need. If it does we + # should switch to clang for MinGW as well! + # + # Note that the LLVM installer is an NSIS installer + # + # Original downloaded here came from + # http://releases.llvm.org/7.0.0/LLVM-7.0.0-win64.exe + # That installer was run through `wine` on Linux and then the resulting + # installation directory (found in `$HOME/.wine/drive_c/Program Files/LLVM`) was + # packaged up into a tarball. We've had issues otherwise that the installer will + # randomly hang, provide not a lot of useful information, pollute global state, + # etc. In general the tarball is just more confined and easier to deal with when + # working with various CI environments. + + mkdir -p citools + cd citools + curl -f "${MIRRORS_BASE}/LLVM-7.0.0-win64.tar.gz" | tar xzf - + ciCommandSetEnv RUST_CONFIGURE_ARGS \ + "${RUST_CONFIGURE_ARGS} --set llvm.clang-cl=$(pwd)/clang-rust/bin/clang-cl.exe" +fi diff --git a/src/ci/scripts/install-innosetup.sh b/src/ci/scripts/install-innosetup.sh new file mode 100755 index 0000000000000..04ca249777a11 --- /dev/null +++ b/src/ci/scripts/install-innosetup.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# We use InnoSetup and its `iscc` program to also create combined installers. +# Honestly at this point WIX above and `iscc` are just holdovers from +# oh-so-long-ago and are required for creating installers on Windows. I think +# one is MSI installers and one is EXE, but they're not used so frequently at +# this point anyway so perhaps it's a wash! + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + curl.exe -o is-install.exe "${MIRRORS_BASE}/2017-08-22-is.exe" + cmd.exe //c "is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP-" + + ciCommandAddPath "C:\\Program Files (x86)\\Inno Setup 5" +fi diff --git a/src/ci/scripts/install-mingw.sh b/src/ci/scripts/install-mingw.sh new file mode 100755 index 0000000000000..b4e8b889f520a --- /dev/null +++ b/src/ci/scripts/install-mingw.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# If we need to download a custom MinGW, do so here and set the path +# appropriately. +# +# Here we also do a pretty heinous thing which is to mangle the MinGW +# installation we just downloaded. Currently, as of this writing, we're using +# MinGW-w64 builds of gcc, and that's currently at 6.3.0. We use 6.3.0 as it +# appears to be the first version which contains a fix for #40546, builds +# randomly failing during LLVM due to ar.exe/ranlib.exe failures. +# +# Unfortunately, though, 6.3.0 *also* is the first version of MinGW-w64 builds +# to contain a regression in gdb (#40184). As a result if we were to use the +# gdb provided (7.11.1) then we would fail all debuginfo tests. +# +# In order to fix spurious failures (pretty high priority) we use 6.3.0. To +# avoid disabling gdb tests we download an *old* version of gdb, specifically +# that found inside the 6.2.0 distribution. We then overwrite the 6.3.0 gdb +# with the 6.2.0 gdb to get tests passing. +# +# Note that we don't literally overwrite the gdb.exe binary because it appears +# to just use gdborig.exe, so that's the binary we deal with instead. +# +# Otherwise install MinGW through `pacman` + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + if [[ -z "${MINGW_URL+x}" ]]; then + arch=i686 + if [ "$MSYS_BITS" = "64" ]; then + arch=x86_64 + fi + pacman -S --noconfirm --needed mingw-w64-$arch-toolchain mingw-w64-$arch-cmake \ + mingw-w64-$arch-gcc mingw-w64-$arch-python2 + ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/mingw${MSYS_BITS}/bin" + else + curl -o mingw.7z "${MINGW_URL}/${MINGW_ARCHIVE}" + 7z x -y mingw.7z > /dev/null + curl -o "${MINGW_DIR}/bin/gdborig.exe" "${MINGW_URL}/2017-04-20-${MSYS_BITS}bit-gdborig.exe" + ciCommandAddPath "$(pwd)/${MINGW_DIR}/bin" + fi +fi diff --git a/src/ci/scripts/install-msys2-packages.sh b/src/ci/scripts/install-msys2-packages.sh new file mode 100755 index 0000000000000..375f13f30b3ed --- /dev/null +++ b/src/ci/scripts/install-msys2-packages.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + pacman -S --noconfirm --needed base-devel ca-certificates make diffutils tar + + # FIXME(#65767): workaround msys bug, step 2 + arch=i686 + if [ "$MSYS_BITS" = "64" ]; then + arch=x86_64 + fi + pacman -U --noconfirm --noprogressbar mingw-w64-$arch-ca-certificates-20180409-1-any.pkg.tar.xz + rm mingw-w64-$arch-ca-certificates-20180409-1-any.pkg.tar.xz + + # Make sure we use the native python interpreter instead of some msys equivalent + # one way or another. The msys interpreters seem to have weird path conversions + # baked in which break LLVM's build system one way or another, so let's use the + # native version which keeps everything as native as possible. + cp C:/Python27amd64/python.exe C:/Python27amd64/python2.7.exe + ciCommandAddPath "C:\\Python27amd64" +fi diff --git a/src/ci/scripts/install-msys2.sh b/src/ci/scripts/install-msys2.sh new file mode 100755 index 0000000000000..8b631192ea2b4 --- /dev/null +++ b/src/ci/scripts/install-msys2.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# ignore-tidy-linelength +# Download and install MSYS2, needed primarily for the test suite (run-make) but +# also used by the MinGW toolchain for assembling things. +# +# FIXME: we should probe the default azure image and see if we can use the MSYS2 +# toolchain there. (if there's even one there). For now though this gets the job +# done. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + # FIXME(#65767): workaround msys bug, step 1 + arch=i686 + if [ "$MSYS_BITS" = "64" ]; then + arch=x86_64 + fi + curl -O "${MIRRORS_BASE}/msys2-repo/mingw/$arch/mingw-w64-$arch-ca-certificates-20180409-1-any.pkg.tar.xz" + + choco install msys2 --params="/InstallDir:${SYSTEM_WORKFOLDER}/msys2 /NoPath" -y --no-progress + mkdir -p "${SYSTEM_WORKFOLDER}/msys2/home/${USERNAME}" + + ciCommandAddPath "${SYSTEM_WORKFOLDER}/msys2/usr/bin" +fi diff --git a/src/ci/scripts/install-ninja.sh b/src/ci/scripts/install-ninja.sh new file mode 100755 index 0000000000000..b8261d8a6f284 --- /dev/null +++ b/src/ci/scripts/install-ninja.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Note that this is originally from the github releases patch of Ninja + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + mkdir ninja + curl -o ninja.zip "${MIRRORS_BASE}/2017-03-15-ninja-win.zip" + 7z x -oninja ninja.zip + rm ninja.zip + ciCommandSetEnv "RUST_CONFIGURE_ARGS" "${RUST_CONFIGURE_ARGS} --enable-ninja" + ciCommandAddPath "$(pwd)/ninja" +fi diff --git a/src/ci/scripts/install-sccache.sh b/src/ci/scripts/install-sccache.sh new file mode 100755 index 0000000000000..d3c298992254e --- /dev/null +++ b/src/ci/scripts/install-sccache.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# This script installs sccache on the local machine. Note that we don't install +# sccache on Linux since it's installed elsewhere through all the containers. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isMacOS; then + curl -fo /usr/local/bin/sccache "${MIRRORS_BASE}/2018-04-02-sccache-x86_64-apple-darwin" + chmod +x /usr/local/bin/sccache +elif isWindows; then + mkdir -p sccache + curl -fo sccache/sccache.exe "${MIRRORS_BASE}/2018-04-26-sccache-x86_64-pc-windows-msvc" + ciCommandAddPath "$(pwd)/sccache" +fi + +# FIXME: we should probably install sccache outside the containers and then +# mount it inside the containers so we can centralize all installation here. diff --git a/src/ci/scripts/install-wix.sh b/src/ci/scripts/install-wix.sh new file mode 100755 index 0000000000000..688f1a49cbfdc --- /dev/null +++ b/src/ci/scripts/install-wix.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# We use the WIX toolset to create combined installers for Windows, and these +# binaries are downloaded from https://github.com/wixtoolset/wix3 originally + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + ciCommandSetEnv WIX "$(pwd)/wix" + + curl -O "${MIRRORS_BASE}/wix311-binaries.zip" + mkdir -p wix/bin + cd wix/bin + 7z x ../../wix311-binaries.zip +fi diff --git a/src/ci/scripts/switch-xcode.sh b/src/ci/scripts/switch-xcode.sh new file mode 100755 index 0000000000000..2cbb2ddbc7046 --- /dev/null +++ b/src/ci/scripts/switch-xcode.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# Switch to XCode 9.3 on OSX since it seems to be the last version that supports +# i686-apple-darwin. We'll eventually want to upgrade this and it will probably +# force us to drop i686-apple-darwin, but let's keep the wheels turning for now. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isMacOS; then + sudo xcode-select --switch /Applications/Xcode_9.3.app +fi diff --git a/src/ci/scripts/verify-line-endings.sh b/src/ci/scripts/verify-line-endings.sh new file mode 100755 index 0000000000000..f3cac13ea4802 --- /dev/null +++ b/src/ci/scripts/verify-line-endings.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# See also the disable for autocrlf, this just checks that it worked. +# +# We check both in rust-lang/rust and in a submodule to make sure both are +# accurate. Submodules are checked out significantly later than the main +# repository in this script, so settings can (and do!) change between then. +# +# Linux (and maybe macOS) builders don't currently have dos2unix so just only +# run this step on Windows. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + # print out the git configuration so we can better investigate failures in + # the following + git config --list --show-origin + dos2unix -ih Cargo.lock src/tools/rust-installer/install-template.sh + endings=$(dos2unix -ic Cargo.lock src/tools/rust-installer/install-template.sh) + # if endings has non-zero length, error out + if [ -n "$endings" ]; then exit 1 ; fi +fi diff --git a/src/ci/scripts/windows-symlink-build-dir.sh b/src/ci/scripts/windows-symlink-build-dir.sh new file mode 100755 index 0000000000000..e57128c70f5f1 --- /dev/null +++ b/src/ci/scripts/windows-symlink-build-dir.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# We've had issues with the default drive in use running out of space during a +# build, and it looks like the `C:` drive has more space than the default `D:` +# drive. We should probably confirm this with the azure pipelines team at some +# point, but this seems to fix our "disk space full" problems. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +if isWindows; then + cmd //c "mkdir c:\\MORE_SPACE" + cmd //c "mklink /J build c:\\MORE_SPACE" +fi diff --git a/src/ci/shared.sh b/src/ci/shared.sh index b093a07ec5c5a..37e45b5639dc9 100644 --- a/src/ci/shared.sh +++ b/src/ci/shared.sh @@ -4,6 +4,8 @@ # `source shared.sh`, hence the invalid shebang and not being # marked as an executable file in git. +export MIRRORS_BASE="https://rust-lang-ci-mirrors.s3-us-west-1.amazonaws.com/rustc" + # See http://unix.stackexchange.com/questions/82598 # Duplicated in docker/dist-various-2/shared.sh function retry { @@ -28,10 +30,39 @@ function isCI { [ "$CI" = "true" ] || [ "$TF_BUILD" = "True" ] } -function isOSX { +function isMacOS { [ "$AGENT_OS" = "Darwin" ] } +function isWindows { + [ "$AGENT_OS" = "Windows_NT" ] +} + +function isLinux { + [ "$AGENT_OS" = "Linux" ] +} + function getCIBranch { echo "$BUILD_SOURCEBRANCHNAME" } + +function ciCommandAddPath { + if [[ $# -ne 1 ]]; then + echo "usage: $0 " + exit 1 + fi + path="$1" + + echo "##vso[task.prependpath]${path}" +} + +function ciCommandSetEnv { + if [[ $# -ne 2 ]]; then + echo "usage: $0 " + exit 1 + fi + name="$1" + value="$2" + + echo "##vso[task.setvariable variable=${name}]${value}" +}