Skip to content

Commit eda5ac0

Browse files
DilumAluthgeDilumAluthge
committed
CI (Buildkite): Add a once-daily scheduled job that builds Julia with USE_BINARYBUILDER=0 (and runs the test suite) on linux64
1 parent e6d0465 commit eda5ac0

File tree

11 files changed

+93
-64
lines changed

11 files changed

+93
-64
lines changed

‎.buildkite/pipelines/main/launch_unsigned_builders.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ steps:
2222
buildkite-agent pipeline upload .buildkite/pipelines/main/misc/sanitizers.yml
2323
2424
# Launch all of the platform jobs.
25-
bash .buildkite/pipelines/main/platforms/platforms.sh package_linux
26-
bash .buildkite/pipelines/main/platforms/platforms.sh tester_linux
25+
bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
26+
bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/main/platforms/tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
2727
2828
# Launch the `whitespace` job last. Uploading it last actually causes it to start
2929
# first. We want this job to start first because we want it to finish as quickly

‎.buildkite/pipelines/main/platforms/package_linux.arches

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
1-
# PLATFORM ARCH ARCH_LABEL ROOTFS_ARCH ALLOW_FAIL TIMEOUT ROOTFS_TAG ROOTFS_TREE
2-
# linux aarch64 aarch64 aarch64 false 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff
3-
# linux armv7l armv7l armv7l false 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25
4-
linux 32 32 i686 false 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32
5-
# linux ppc64le ppc64le powerpc64le false 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7
6-
linux 64 64 x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
7-
musl 64 64 x86_64 false 60 v3.19 e6a2730e37c386c46915b2650d6aaaa398195152
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
# linux _aarch64 false _aarch64 aarch64 none 60 no no no v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff
3+
# linux _armv7l false _armv7l armv7l none 60 no no no v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25
4+
linux 32 false 32 i686 none 60 no no no v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32
5+
# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7
6+
linux 64 false 64 x86_64 none 60 no no no v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
7+
musl 64 false 64 x86_64 none 60 no no no v3.19 e6a2730e37c386c46915b2650d6aaaa398195152

‎.buildkite/pipelines/main/platforms/package_linux.yml

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,16 +4,16 @@ agents:
44
sandbox.jl: "true"
55
os: "linux"
66
steps:
7-
- label: "package_${PLATFORM?}${ARCH_LABEL?}"
8-
key: package_${PLATFORM?}${ARCH_LABEL?}
7+
- label: "package_${PLATFORM?}${LABEL?}"
8+
key: package_${PLATFORM?}${LABEL?}
99
plugins:
1010
- JuliaCI/julia#v1:
1111
# Drop default "registries" directory, so it is not persisted from execution to execution
1212
persist_depot_dirs: packages,artifacts,compiled
1313
version: '1.6'
1414
- staticfloat/sandbox#v1:
15-
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
16-
rootfs_treehash: "${ROOTFS_TREE?}"
15+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
16+
rootfs_treehash: "${ROOTFS_HASH?}"
1717
uid: 1000
1818
gid: 1000
1919
workspaces:
@@ -26,7 +26,7 @@ steps:
2626
SHORT_COMMIT=`echo $${BUILDKITE_COMMIT:?} | cut -c1-$${SHORT_COMMIT_LENGTH:?}`
2727
ARTIFACT_FILE_EXTENSION="tar.gz"
2828
ARTIFACT_FILENAME="julia-$${SHORT_COMMIT:?}-${PLATFORM?}${ARCH?}.$${ARTIFACT_FILE_EXTENSION:?}"
29-
JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME | cut -c27-`
29+
JULIA_BINARYDIST_FILENAME=`make print-JULIA_BINARYDIST_FILENAME ${MAKE_FLAGS?} | cut -c27- | tr -s ' '`
3030
JULIA_BINARYDIST="$${JULIA_BINARYDIST_FILENAME:?}.$${ARTIFACT_FILE_EXTENSION:?}"
3131
3232
echo "The full commit is: $${BUILDKITE_COMMIT:?}"
@@ -35,16 +35,16 @@ steps:
3535
3636
echo "--- Build Julia from source"
3737
rm -rf $${ARTIFACT_FILENAME:?}
38-
make --output-sync -j 8
38+
make --output-sync -j 8 ${MAKE_FLAGS?}
3939
40-
echo "--- Make sure that the working directory is clean"
40+
echo "--- Check that the working directory is clean"
4141
if [ -z "$(git status --short)" ]; then echo "INFO: The working directory is clean."; else echo "ERROR: The working directory is dirty."; echo "Output of git status:"; git status; exit 1; fi
4242
4343
echo "--- Print Julia version info"
4444
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
4545
4646
echo "--- Create build artifacts"
47-
make --output-sync -j 8 binary-dist
47+
make --output-sync -j 8 binary-dist ${MAKE_FLAGS?}
4848
ls -l $${JULIA_BINARYDIST:?}
4949
if [[ "$${JULIA_BINARYDIST:?}" != "$${ARTIFACT_FILENAME:?}" ]]; then
5050
mv $${JULIA_BINARYDIST:?} $${ARTIFACT_FILENAME:?}

‎.buildkite/pipelines/main/platforms/platforms.sh

Lines changed: 0 additions & 26 deletions
This file was deleted.

‎.buildkite/pipelines/main/platforms/tester_linux.arches

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
# PLATFORM ARCH ARCH_LABEL ROOTFS_ARCH ALLOW_FAIL TIMEOUT ROOTFS_TAG ROOTFS_TREE
2-
# linux aarch64 aarch64 aarch64 false 60 v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff
3-
# linux armv7l armv7l armv7l false 60 v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25
4-
linux 32 32 i686 false 60 v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32
5-
# linux ppc64le ppc64le powerpc64le false 60 v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7
6-
linux 64 64_rr x86_64 false 180 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
7-
linux 64 64_st x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
8-
linux 64 64_mt x86_64 false 60 v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
9-
musl 64 64 x86_64 true 60 v3.19 e6a2730e37c386c46915b2650d6aaaa398195152
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
# linux _aarch64 false _aarch64 aarch64 none 60 no no no v3.2 0566841e29f0f9880541c26a6595fd5ce0beb5ff
3+
# linux _armv7l false _armv7l armv7l none 60 no no no v3.2 fb359370b052a47ce5c84cc6b4a7a03ed7053b25
4+
linux 32 false 32 i686 none 60 no no no v3.2 209c4db679a515befd7fb50ecc6bfbecf7ec3d32
5+
# linux _ppc64le false _ppc64le powerpc64le none 60 no no no v3.2 c03a0158b19d48ac84b426834fce0d3584cdd0c7
6+
linux 64_rr false 64 x86_64 none 60 yes no no v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
7+
linux 64_st false 64 x86_64 none 60 no yes no v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
8+
linux 64_mt false 64 x86_64 none 60 no no yes v3.2 474bf61a926b2d7fcf202284d59d4b11a04601d7
9+
musl 64 true 64 x86_64 none 60 no no no v3.19 e6a2730e37c386c46915b2650d6aaaa398195152

‎.buildkite/pipelines/main/platforms/tester_linux.yml

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,18 @@ agents:
44
sandbox.jl: "true"
55
os: "linux"
66
steps:
7-
- label: "tester_${PLATFORM?}${ARCH_LABEL?}"
8-
key: tester_${PLATFORM?}${ARCH_LABEL?}
7+
- label: "tester_${PLATFORM?}${LABEL?}"
8+
key: tester_${PLATFORM?}${LABEL?}
99
depends_on: package_${PLATFORM?}${ARCH?}
1010
plugins:
1111
- JuliaCI/julia#v1:
1212
# Drop default "registries" directory, so it is not persisted from execution to execution
1313
persist_depot_dirs: packages,artifacts,compiled
1414
version: '1.6'
1515
- staticfloat/sandbox#v1:
16-
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
17-
# rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ROOTFS_ARCH?}.tar.gz
18-
rootfs_treehash: "${ROOTFS_TREE?}"
16+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/package_${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
17+
# rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/${ROOTFS_TAG?}/tester${PLATFORM?}.${ARCH_ROOTFS?}.tar.gz
18+
rootfs_treehash: "${ROOTFS_HASH?}"
1919
uid: 1000
2020
gid: 1000
2121
workspaces:
@@ -57,7 +57,7 @@ steps:
5757
export OPENBLAS_NUM_THREADS=8
5858
export TESTS="[\"all\"]"
5959
60-
if [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_rr" ]]; then
60+
if [[ "${IS_RR?}" == "yes" ]]; then
6161
export JULIA_BINARY_UNDER_RR="$${JULIA_BINARY:?} .buildkite/utilities/rr/rr_capture.jl $${JULIA_BINARY:?}"
6262
6363
export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY_UNDER_RR:?}"
@@ -69,11 +69,11 @@ steps:
6969
export JULIA_BINARY_FOR_TESTS="$${JULIA_BINARY:?}"
7070
export NCORES_FOR_TESTS="Sys.CPU_THREADS"
7171
72-
if [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_st" ]]; then
73-
# "_st" = single-threaded
72+
if [[ "${IS_ST?}" == "yes" ]]; then
73+
# "ST" = single-threaded
7474
export JULIA_NUM_THREADS=1
75-
elif [[ "$${BUILDKITE_STEP_KEY:?}" == "tester_linux64_mt" ]]; then
76-
# "_mt" = multi-threaded
75+
elif [[ "${IS_MT?}" == "yes" ]]; then
76+
# "MT" = multi-threaded
7777
export JULIA_NUM_THREADS=16
7878
7979
# We do not run the Distributed test suite when multithreading is enabled.

‎.buildkite/pipelines/scheduled/0_webui.yml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,8 @@ steps:
1313
# Build secrets will not be available in these pipelines (or their children)
1414
# but some of our signed pipelines can wait upon the completion of these unsigned
1515
# pipelines.
16-
# unsigned_pipelines:
17-
# - .buildkite/pipelines/scheduled/launch_unsigned_builders.yml
16+
unsigned_pipelines:
17+
- .buildkite/pipelines/scheduled/launch_unsigned_jobs.yml
1818

1919
# Our signed pipelines must have a `signature` or `signature_file` parameter that
2020
# verifies the treehash of the pipeline itself and the inputs listed in `inputs`

‎.buildkite/pipelines/scheduled/launch_unsigned_jobs.yml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
steps:
2+
- label: ":buildkite: Launch unsigned jobs"
3+
commands: |
4+
# Launch all of the `USE_BINARYBUILDER=0` jobs.
5+
bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches .buildkite/pipelines/main/platforms/package_linux.yml
6+
bash .buildkite/utilities/platforms/platforms.sh .buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches .buildkite/pipelines/main/platforms/tester_linux.yml
7+
agents:
8+
queue: julia

‎.buildkite/pipelines/scheduled/no_bb/no_bb_package_linux.arches

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
linux 64_no_bb false 64_no_bb x86_64 USE_BINARYBUILDER=0 240 no no no v4.4 ec3873fc1756ecb4c815e8785e9ca45cf54c68b6

‎.buildkite/pipelines/scheduled/no_bb/no_bb_tester_linux.arches

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
# PLATFORM LABEL ALLOW_FAIL ARCH ARCH_ROOTFS MAKE_FLAGS TIMEOUT IS_RR IS_ST IS_MT ROOTFS_TAG ROOTFS_HASH
2+
linux 64_st_no_bb false 64_no_bb x86_64 none 60 no yes no v4.4 ec3873fc1756ecb4c815e8785e9ca45cf54c68b6

0 commit comments

Comments
 (0)