Skip to content

Commit e352f5a

Browse files
fanquakefanquake
committed
Merge bitcoin#27028: ci: Cache more stuff in the ci images: msan, iwyu, pip, sdks
faa0839 ci: Cache more stuff in the ci images: msan, iwyu, pip, sdks (MarcoFalke) Pull request description: Now that `apt` packages are cached in the ci images, it makes sense to think about caching all other packages as well. ACKs for top commit: TheCharlatan: re-ACK faa0839 Tree-SHA512: e2ea491570c6cdcc8522585ae7669c51ab2c0b680ff34067b58727994aa8f2e5c45ba7b76ed27a9c76d788ed155d7aade554dc164f7552fa713c00cc47b722f1
2 parents 873a506 + faa0839 commit e352f5a

File tree

4 files changed

+62
-60
lines changed

4 files changed

+62
-60
lines changed

‎ci/test/00_setup_env.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build}
6565
# The folder for previous release binaries.
6666
# This folder exists only on the ci guest, and on the ci host as a volume.
6767
export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST}
68+
export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
6869
export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
6970
export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison}
7071
export GOAL=${GOAL:-install}

‎ci/test/01_base_install.sh‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,13 @@
66

77
export LC_ALL=C.UTF-8
88

9+
# This script is always run as root, and the functions can be removed and
10+
# replaced by a bash -c "command" directly, where needed.
11+
# For now, they are named aliases to aid review with --color-moved=dimmed-zebra.
912
CI_EXEC_ROOT () { bash -c "$*"; }
13+
CI_EXEC() { bash -c "$*"; }
1014
export -f CI_EXEC_ROOT
15+
export -f CI_EXEC
1116

1217
if [ -n "$DPKG_ADD_ARCH" ]; then
1318
CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH"
@@ -31,3 +36,55 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then
3136
${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update
3237
${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES"
3338
fi
39+
40+
if [ -n "$PIP_PACKAGES" ]; then
41+
if [ "$CI_OS_NAME" == "macos" ]; then
42+
sudo -H pip3 install --upgrade pip
43+
# shellcheck disable=SC2086
44+
IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
45+
else
46+
# shellcheck disable=SC2086
47+
${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
48+
fi
49+
fi
50+
51+
if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
52+
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
53+
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
54+
CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
55+
CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
56+
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
57+
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
58+
fi
59+
60+
if [[ "${RUN_TIDY}" == "true" ]]; then
61+
if [ ! -d "${DIR_IWYU}" ]; then
62+
CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
63+
CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use"
64+
CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use"
65+
CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
66+
fi
67+
fi
68+
69+
CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
70+
71+
OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
72+
73+
if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
74+
OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
75+
OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
76+
if [ ! -f "$OSX_SDK_PATH" ]; then
77+
CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
78+
fi
79+
CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
80+
fi
81+
82+
if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
83+
ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
84+
if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
85+
CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
86+
fi
87+
CI_EXEC mkdir -p "$ANDROID_HOME"
88+
CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
89+
CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
90+
fi

‎ci/test/04_install.sh‎

Lines changed: 4 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,10 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
5959
--name $CONTAINER_NAME \
6060
$CONTAINER_NAME)
6161
export CI_CONTAINER_ID
62+
export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID"
63+
export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
64+
$CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ci_base_install/ "${BASE_ROOT_DIR}"
65+
$CI_EXEC_CMD_PREFIX_ROOT rsync --archive --stats --human-readable /ro_base/ "$BASE_ROOT_DIR"
6266

6367
# Create a non-root user inside the container which matches the local user.
6468
#
@@ -67,8 +71,6 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
6771
docker exec "$CI_CONTAINER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER"
6872
docker exec "$CI_CONTAINER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER"
6973
docker exec "$CI_CONTAINER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}"
70-
export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID"
71-
export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID"
7274
else
7375
echo "Running on host system without docker wrapper"
7476
"${BASE_ROOT_DIR}/ci/test/01_base_install.sh"
@@ -85,17 +87,6 @@ export -f CI_EXEC_ROOT
8587

8688
CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}"
8789

88-
if [ -n "$PIP_PACKAGES" ]; then
89-
if [ "$CI_OS_NAME" == "macos" ]; then
90-
sudo -H pip3 install --upgrade pip
91-
# shellcheck disable=SC2086
92-
IN_GETOPT_BIN="$(brew --prefix gnu-getopt)/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES
93-
else
94-
# shellcheck disable=SC2086
95-
${CI_RETRY_EXE} CI_EXEC pip3 install --user $PIP_PACKAGES
96-
fi
97-
fi
98-
9990
if [ "$CI_OS_NAME" == "macos" ]; then
10091
top -l 1 -s 0 | awk ' /PhysMem/ {print}'
10192
echo "Number of CPUs: $(sysctl -n hw.logicalcpu)"
@@ -122,30 +113,6 @@ fi
122113

123114
CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
124115

125-
if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then
126-
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100"
127-
CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100"
128-
CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/"
129-
CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project"
130-
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/"
131-
CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && make $MAKEJOBS cxx"
132-
fi
133-
134-
if [[ "${RUN_TIDY}" == "true" ]]; then
135-
export DIR_IWYU="${BASE_SCRATCH_DIR}/iwyu"
136-
if [ ! -d "${DIR_IWYU}" ]; then
137-
CI_EXEC "mkdir -p ${DIR_IWYU}/build/"
138-
CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_15 ${DIR_IWYU}/include-what-you-use"
139-
CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-15 ../include-what-you-use"
140-
CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS"
141-
fi
142-
fi
143-
144-
if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then
145-
echo "Create $BASE_ROOT_DIR"
146-
CI_EXEC rsync -a /ro_base/ "$BASE_ROOT_DIR"
147-
fi
148-
149116
if [ "$USE_BUSY_BOX" = "true" ]; then
150117
echo "Setup to use BusyBox utils"
151118
# tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version)

‎ci/test/05_before_script.sh‎

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -14,29 +14,6 @@ else
1414
CI_EXEC_ROOT echo \> \$HOME/.bitcoin
1515
fi
1616

17-
CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources"
18-
19-
OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers"
20-
21-
if [ -n "$XCODE_VERSION" ] && [ ! -d "${DEPENDS_DIR}/SDKs/${OSX_SDK_BASENAME}" ]; then
22-
OSX_SDK_FILENAME="${OSX_SDK_BASENAME}.tar.gz"
23-
OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_FILENAME}"
24-
if [ ! -f "$OSX_SDK_PATH" ]; then
25-
CI_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_FILENAME}" -o "$OSX_SDK_PATH"
26-
fi
27-
CI_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
28-
fi
29-
30-
if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then
31-
ANDROID_TOOLS_PATH=${DEPENDS_DIR}/sdk-sources/android-tools.zip
32-
if [ ! -f "$ANDROID_TOOLS_PATH" ]; then
33-
CI_EXEC curl --location --fail "${ANDROID_TOOLS_URL}" -o "$ANDROID_TOOLS_PATH"
34-
fi
35-
CI_EXEC mkdir -p "$ANDROID_HOME"
36-
CI_EXEC unzip -o "$ANDROID_TOOLS_PATH" -d "$ANDROID_HOME"
37-
CI_EXEC "yes | ${ANDROID_HOME}/cmdline-tools/bin/sdkmanager --sdk_root=\"${ANDROID_HOME}\" --install \"build-tools;${ANDROID_BUILD_TOOLS_VERSION}\" \"platform-tools\" \"platforms;android-${ANDROID_API_LEVEL}\" \"ndk;${ANDROID_NDK_VERSION}\""
38-
fi
39-
4017
if [ -z "$NO_DEPENDS" ]; then
4118
if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then
4219
# CentOS has problems building the depends if the config shell is not explicitly set

0 commit comments

Comments
 (0)