Skip to content

Commit 4fe53ea

Browse files
DilumAluthgestaticfloat
DilumAluthge
authored and
staticfloat
committed
CI (Buildkite): add a separate Buildkite job that runs the tests of the network-related stdlibs (e.g. Downloads.jl), and automatically retries that job up to a maximum number of tries (#42861)
(cherry picked from commit 3003742)
1 parent 177cf61 commit 4fe53ea

File tree

8 files changed

+112
-69
lines changed

8 files changed

+112
-69
lines changed

‎.buildkite/pipelines/main/platforms/package_linux.arches‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2-
# linux _aarch64 false _aarch64 aarch64 none 60 no no no v0.0 0000000000000000000000000000000000000000
3-
# linux _armv7l false _armv7l armv7l none 60 no no no v0.0 0000000000000000000000000000000000000000
4-
linux 32 false 32 i686 none 60 no no no v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
5-
# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v0.0 0000000000000000000000000000000000000000
6-
linux 64 false 64 x86_64 none 60 no no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
7-
musl 64 false 64 x86_64 none 60 no no no v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
linux 32 . 32 i686 . . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
3+
linux 64 . 64 x86_64 . . . . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
4+
# linux _aarch64 . _aarch64 aarch64 . . . . . . . . .... ........................................
5+
# linux _armv7l . _armv7l armv7l . . . . . . . . .... ........................................
6+
# linux _ppc64le . _ppc64le powerpc64le . . . . . . . . .... ........................................
7+
musl 64 . 64 x86_64 . . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90

‎.buildkite/pipelines/main/platforms/package_linux.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ steps:
1919
workspaces:
2020
# Include `/cache/repos` so that our `git` version introspection works.
2121
- "/cache/repos:/cache/repos"
22-
timeout_in_minutes: ${TIMEOUT?}
22+
timeout_in_minutes: ${TIMEOUT_BK?}
2323
commands: |
2424
echo "--- Print the full and short commit hashes"
2525
SHORT_COMMIT_LENGTH=10

‎.buildkite/pipelines/main/platforms/tester_linux.arches‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,14 @@
1-
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2-
# linux _aarch64 false _aarch64 aarch64 none 60 no no no v0.0 0000000000000000000000000000000000000000
3-
# linux _armv7l false _armv7l armv7l none 60 no no no v0.0 0000000000000000000000000000000000000000
4-
linux 32 false 32 i686 none 60 no no no v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
5-
# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v0.0 0000000000000000000000000000000000000000
6-
linux 64_rr false 64 x86_64 none 180 yes no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
7-
linux 64_st false 64 x86_64 none 60 no yes no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
8-
linux 64_mt false 64 x86_64 none 60 no no yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
9-
musl 64 true 64 x86_64 none 60 no no no v4.8 d13a47c87c38005bd5d97132e51789cafd852f90
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
3+
linux 32 . 32 i686 no-net . . . . . . . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
4+
linux 32_net . 32 i686 net . . . 3 . yes . v4.8 b6dffc772ab4c2cd7fd4f83459308f6f0d89b957
5+
6+
linux 64_rr . 64 x86_64 no-net . 180 120 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
7+
linux 64_net . 64 x86_64 net . 120 60 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
8+
linux 64_st . 64 x86_64 no-net . . . . . yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
9+
linux 64_mt . 64 x86_64 no-net . . . . . . yes v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
10+
11+
# linux _aarch64 . _aarch64 aarch64 . . . . . . . . .... ........................................
12+
# linux _armv7l . _armv7l armv7l . . . . . . . . .... ........................................
13+
# linux _ppc64le . _ppc64le powerpc64le . . . . . . . . .... ........................................
14+
musl 64 true 64 x86_64 . . . . . . . . v4.8 d13a47c87c38005bd5d97132e51789cafd852f90

‎.buildkite/pipelines/main/platforms/tester_linux.yml‎

Lines changed: 30 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,11 @@ steps:
2323
- "/cache/repos:/cache/repos"
2424
env:
2525
JULIA_SHELL: "/bin/bash"
26-
timeout_in_minutes: ${TIMEOUT?}
26+
timeout_in_minutes: ${TIMEOUT_BK?}
27+
retry:
28+
automatic:
29+
- exit_status: "*"
30+
limit: ${RETRIES?}
2731
soft_fail: ${ALLOW_FAIL?}
2832
commands: |
2933
echo "--- Print the full and short commit hashes"
@@ -55,45 +59,42 @@ steps:
5559
echo "--- Set some environment variables in preparation for running the Julia test suite"
5660
unset JULIA_DEPOT_PATH
5761
export OPENBLAS_NUM_THREADS=8
58-
export TESTS="[\"all\"]"
5962
60-
if [[ "${IS_RR?}" == "yes" ]]; then
61-
export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
63+
if [[ "${GROUP?}" == "all" ]]; then
64+
export TESTS="all"
65+
elif [[ "${GROUP?}" == "net" ]]; then
66+
export TESTS="Downloads LibGit2 LibGit2/online download"
67+
elif [[ "${GROUP?}" == "no-net" ]]; then
68+
export TESTS="all --skip Downloads LibGit2/online download"
69+
else
70+
echo "Invalid value for GROUP: ${GROUP?}"
71+
exit 1
72+
fi
6273
63-
export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}"
64-
export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
74+
export JULIA_TEST_RR_TIMEOUT="${TIMEOUT_RR?}"
6575
66-
# For the `rr` job, we disable multi-threading.
67-
export JULIA_NUM_THREADS=1
76+
if [[ "${IS_RR?}" == "yes" ]]; then
77+
export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
78+
export NCORES_FOR_TESTS="parse(Int, ENV[\"JULIA_RRCAPTURE_NUM_CORES\"])"
6879
else
69-
export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}"
80+
export JULIA_CMD_FOR_TESTS="$${JULIA_BINARY:?}"
7081
export NCORES_FOR_TESTS="Sys.CPU_THREADS"
82+
fi
7183
72-
if [[ "${IS_ST?}" == "yes" ]]; then
73-
# "ST" = single-threaded
74-
export JULIA_NUM_THREADS=1
75-
elif [[ "${IS_MT?}" == "yes" ]]; then
76-
# "MT" = multi-threaded
77-
export JULIA_NUM_THREADS=16
84+
if [[ "${IS_ST?}" == "yes" ]]; then
85+
export JULIA_NUM_THREADS=1
86+
fi
7887
79-
if [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-6" ]]; then
80-
# On Julia 1.6, we skip the Distributed test suite if multithreading is enabled.
81-
export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
82-
elif [[ "$${BUILDKITE_PIPELINE_SLUG:?}" == "julia-release-1-dot-7" ]]; then
83-
# On Julia 1.7, we skip the Distributed test suite if multithreading is enabled.
84-
export TESTS="[\"all\", \"--skip\", \"Distributed\"]"
85-
else
86-
# On Julia 1.8 and later, we do not skip any test suites.
87-
export TESTS="[\"all\"]"
88-
fi
89-
fi
88+
if [[ "${IS_MT?}" == "yes" ]]; then
89+
export JULIA_NUM_THREADS=16
9090
fi
9191
92-
echo "JULIA_BINARY_FOR_TESTS is: $${JULIA_BINARY_FOR_TESTS:?}"
93-
echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # this variable might not be set
92+
echo "JULIA_CMD_FOR_TESTS is: $${JULIA_CMD_FOR_TESTS:?}"
93+
echo "JULIA_NUM_THREADS is: $${JULIA_NUM_THREADS}" # Note: this environment variable might not be set
9494
echo "NCORES_FOR_TESTS is: $${NCORES_FOR_TESTS:?}"
9595
echo "OPENBLAS_NUM_THREADS is: $${OPENBLAS_NUM_THREADS:?}"
96+
echo "GROUP is: ${GROUP?}"
9697
echo "TESTS is: $${TESTS:?}"
9798
9899
echo "--- Run the Julia test suite"
99-
$${JULIA_BINARY_FOR_TESTS:?} -e "Base.runtests($${TESTS:?}; ncores = $${NCORES_FOR_TESTS:?})"
100+
$${JULIA_CMD_FOR_TESTS:?} -e "Base.runtests(\"$${TESTS:?}\"; ncores = $${NCORES_FOR_TESTS:?})"

‎.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2-
linux 64_no_bb false 64_no_bb x86_64 USE_BINARYBUILDER=0 240 no no no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
linux 64_no_bb false 64_no_bb x86_64 . USE_BINARYBUILDER=0 180 . . . . . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

‎.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
1-
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2-
linux 64_st_no_bb false 64_no_bb x86_64 none 60 no yes no v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS GROUP MAKE_FLAGS TIMEOUT_BK TIMEOUT_RR RETRIES IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
linux 64_no_bb . 64_no_bb x86_64 no-net . 180 120 . yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a
3+
linux 64_no_bb_net . 64_no_bb x86_64 net . 180 120 3 yes yes . v4.8 2a058481b567f0e91b9aa3ce4ad4f09e6419355a

‎.buildkite/utilities/platforms/platforms.sh‎

Lines changed: 47 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -4,40 +4,75 @@ ARCHES="$1"
44
YAML="$2"
55

66
if [[ ! -f "${ARCHES:?}" ]] ; then
7-
echo "File does not exist: ${ARCHES:?}"
7+
echo "Arches file does not exist: ${ARCHES:?}"
88
exit 1
99
fi
1010

1111
if [[ ! -f "${YAML:?}" ]] ; then
12-
echo "File does not exist: ${YAML:?}"
12+
echo "YAML file does not exist: ${YAML:?}"
1313
exit 1
1414
fi
1515

1616
cat "${ARCHES:?}" | tr -s ' ' | while read _line; do
1717
# Remove whitespace from the beginning and end of each line
1818
line=`echo $_line | tr -s ' '`
1919

20-
# Skip all lines that begin with `#`
20+
# Skip any line that begins with the `#` character
2121
if [[ $line == \#* ]]; then
2222
continue
2323
fi
2424

25+
# Skip any empty line
26+
if [[ $line == "" ]]; then
27+
continue
28+
fi
29+
2530
export PLATFORM=`echo $line | cut -d ' ' -f 1 | tr -s ' '`
2631
export LABEL=`echo $line | cut -d ' ' -f 2 | tr -s ' '`
2732
export ALLOW_FAIL=`echo $line | cut -d ' ' -f 3 | tr -s ' '`
2833
export ARCH=`echo $line | cut -d ' ' -f 4 | tr -s ' '`
2934
export ARCH_ROOTFS=`echo $line | cut -d ' ' -f 5 | tr -s ' '`
30-
export MAKE_FLAGS=`echo $line | cut -d ' ' -f 6 | tr -s ' '`
31-
export TIMEOUT=`echo $line | cut -d ' ' -f 7 | tr -s ' '`
32-
export IS_RR=`echo $line | cut -d ' ' -f 8 | tr -s ' '`
33-
export IS_ST=`echo $line | cut -d ' ' -f 9 | tr -s ' '`
34-
export IS_MT=`echo $line | cut -d ' ' -f 10 | tr -s ' '`
35-
export ROOTFS_TAG=`echo $line | cut -d ' ' -f 11 | tr -s ' '`
36-
export ROOTFS_HASH=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
37-
38-
if [[ "${MAKE_FLAGS:?}" == "none" ]]; then
35+
export GROUP=`echo $line | cut -d ' ' -f 6 | tr -s ' '`
36+
export MAKE_FLAGS=`echo $line | cut -d ' ' -f 7 | tr -s ' '`
37+
export TIMEOUT_BK=`echo $line | cut -d ' ' -f 8 | tr -s ' '`
38+
export TIMEOUT_RR=`echo $line | cut -d ' ' -f 9 | tr -s ' '`
39+
export RETRIES=`echo $line | cut -d ' ' -f 10 | tr -s ' '`
40+
export IS_RR=`echo $line | cut -d ' ' -f 11 | tr -s ' '`
41+
export IS_ST=`echo $line | cut -d ' ' -f 12 | tr -s ' '`
42+
export IS_MT=`echo $line | cut -d ' ' -f 13 | tr -s ' '`
43+
export ROOTFS_TAG=`echo $line | cut -d ' ' -f 14 | tr -s ' '`
44+
export ROOTFS_HASH=`echo $line | cut -d ' ' -f 15 | tr -s ' '`
45+
46+
if [[ "${ALLOW_FAIL:?}" == "." ]]; then
47+
export ALLOW_FAIL="false"
48+
fi
49+
50+
if [[ "${GROUP:?}" == "." ]]; then
51+
export GROUP="all"
52+
fi
53+
54+
if [[ "${MAKE_FLAGS:?}" == "." ]]; then
3955
export MAKE_FLAGS=""
4056
fi
4157

58+
if [[ "${TIMEOUT_BK:?}" == "." ]]; then
59+
export TIMEOUT_BK="60"
60+
fi
61+
62+
if [[ "${TIMEOUT_RR:?}" == "." ]]; then
63+
export TIMEOUT_RR="30"
64+
fi
65+
66+
if [[ "${RETRIES:?}" == "." ]]; then
67+
export RETRIES="0"
68+
fi
69+
70+
if [[ "${IS_ST:?}" == "yes" ]]; then
71+
if [[ "${IS_MT:?}" == "yes" ]]; then
72+
echo "You cannot set both IS_ST and IS_MT to yes"
73+
exit 1
74+
fi
75+
fi
76+
4277
buildkite-agent pipeline upload "${YAML:?}"
4378
done

‎.buildkite/utilities/rr/rr_capture.jl‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,14 @@ end
5555

5656
@info "We will run the command under rr"
5757

58-
const build_number = get_from_env("BUILDKITE_BUILD_NUMBER")
59-
const job_name = get_from_env("BUILDKITE_STEP_KEY")
60-
const commit_full = get_from_env("BUILDKITE_COMMIT")
61-
const commit_short = first(commit_full, 10)
62-
const timeout_minutes = 120
63-
const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8")
64-
const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES)
58+
const build_number = get_from_env("BUILDKITE_BUILD_NUMBER")
59+
const job_name = get_from_env("BUILDKITE_STEP_KEY")
60+
const commit_full = get_from_env("BUILDKITE_COMMIT")
61+
const commit_short = first(commit_full, 10)
62+
const JULIA_TEST_RR_TIMEOUT = get(ENV, "JULIA_TEST_RR_TIMEOUT", "120")
63+
const timeout_minutes = parse(Int, JULIA_TEST_RR_TIMEOUT)
64+
const JULIA_TEST_NUM_CORES = get(ENV, "JULIA_TEST_NUM_CORES", "8")
65+
const julia_test_num_cores_int = parse(Int, JULIA_TEST_NUM_CORES)
6566
const num_cores = min(
6667
8,
6768
Sys.CPU_THREADS,

0 commit comments

Comments
 (0)