Skip to content

Commit fd6d294

Browse files
DilumAluthgestaticfloat
authored and
KristofferC
committed
Transition the coverage-linux64 pipeline to Buildkite (#41238)
* Transition the `coverage-linux64` pipeline to Buildkite * Simplify, run inside of a sandbox * Upload coverage reports to Codecov and Coveralls * Add `COVERALLS_TOKEN` Co-authored-by: Elliot Saba <staticfloat@gmail.com> (cherry picked from commit 9d5f31e)
1 parent 2fab90b commit fd6d294

File tree

5 files changed

+217
-0
lines changed

5 files changed

+217
-0
lines changed

‎.buildkite/coverage-linux64/0_webui.yml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
# This file represents what is put into the webUI.
2+
# It is purely for keeping track of the changes we make to the webUI configuration; modifying this file has no effect.
3+
# We use the `cryptic` buildkite plugin to provide secrets management, which requires some integration into the WebUI's steps.
4+
agents:
5+
queue: "julia"
6+
sandbox.jl: "true"
7+
8+
steps:
9+
- label: ":unlock: Unlock secrets, launch pipelines"
10+
plugins:
11+
- staticfloat/cryptic:
12+
# Our signed pipelines must have a `signature` or `signature_file` parameter that
13+
# verifies the treehash of the pipeline itself and the inputs listed in `inputs`
14+
signed_pipelines:
15+
- pipeline: .buildkite/coverage-linux64/pipeline.yml
16+
signature: "U2FsdGVkX18eQWpd3hMYLO5Kd+6K+oBoLk1I6J3qIw7lc6g5/jaeWyq/wralosZCfTzyjS4NstNKFvhQf3KDPEBVElipNvTxoWOjVLRVOrfBqqvTkQN4xVosY/r026Gy"

‎.buildkite/coverage-linux64/README.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
# Coverage pipeline
2+
3+
We run coverage on a separate pipeline, that uses a scheduled build rather than webhooks.
4+
The pipeline is here: https://buildkite.com/julialang/julia-coverage-linux64
5+
6+
It contains [its own webui steps](0_webuiy.ml) (listed here in this repository for clarity) and its own [pipeline.yml](pipeline.yml).

‎.buildkite/coverage-linux64/pipeline.yml

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# These steps should only run on `sandbox.jl` machines, not `docker`-isolated ones
2+
# since we need nestable sandboxing. The rootfs images being used here are built from
3+
# the `.buildkite/rootfs_images/llvm-passes.jl` file.
4+
agents:
5+
queue: "julia"
6+
# Only run on `sandbox.jl` machines (not `docker`-isolated ones) since we need nestable sandboxing
7+
sandbox.jl: "true"
8+
os: "linux"
9+
10+
steps:
11+
- label: ":unlock: :coverage: Run coverage test"
12+
plugins:
13+
- staticfloat/cryptic:
14+
variables:
15+
- CODECOV_TOKEN="U2FsdGVkX19l0fhdBabbuiEdysyEabkJLRHfxm7CNRkuGbnwPV365sxxC7Czs/CVcws0N1oB4pVwALRRMe36oA=="
16+
- COVERALLS_TOKEN="U2FsdGVkX19zopI0hMNzzi2UUOvNVFD8Y0iisFnO/ryVxU7Tit8ZEaeN+gxodRx4CosUUh192F1+q3dTMWRIvw=="
17+
- JuliaCI/julia#v1:
18+
version: 1.6
19+
- staticfloat/sandbox#v1:
20+
rootfs_url: https://github.com/JuliaCI/rootfs-images/releases/download/v1/llvm-passes.tar.gz
21+
rootfs_treehash: "f3ed53f159e8f13edfba8b20ebdb8ece73c1b8a8"
22+
uid: 1000
23+
gid: 1000
24+
commands: |
25+
echo "--- Build Julia from source"
26+
make -j 6
27+
28+
echo "--- Print Julia version info"
29+
./julia -e 'using InteractiveUtils; InteractiveUtils.versioninfo()'
30+
./julia -e '@info "" Sys.CPU_THREADS'
31+
# this is necessary to make sure that the LibGit2 tests passes
32+
git config --global init.defaultBranch master
33+
34+
echo "--- Run Julia tests with code coverage enabled"
35+
# Run the actual tests
36+
./julia --code-coverage=all --sysimage-native-code=no .buildkite/coverage-linux64/run_tests_base.jl
37+
38+
echo "--- Process and upload coverage information"
39+
./julia .buildkite/coverage-linux64/upload_coverage.jl
40+
timeout_in_minutes: 600 # 600 minutes = 10 hours
41+
42+
# We must accept the signed job id secret in order to propagate secrets
43+
env:
44+
BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET: ${BUILDKITE_PLUGIN_CRYPTIC_BASE64_SIGNED_JOB_ID_SECRET?}

‎.buildkite/coverage-linux64/run_tests_base.jl

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# When running this file, make sure to set the `--code-coverage=all` command-line flag.
2+
3+
# Important note: even if one or more tests fail, we will still exit with status code 0.
4+
5+
# The reason for this is that we always want to upload code coverage, even if some of the
6+
# tests fail. Therefore, even if the `coverage-linux64` pipeline passes, you should not
7+
# assume that all of the tests passed. If you want to know if all of the tests are passing,
8+
# please look at the status of the `tester_linux64` pipeline.
9+
10+
const include_tests = String[]
11+
12+
const exclude_tests = String[]
13+
14+
empty!(Base.DEPOT_PATH)
15+
push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
16+
17+
module ChooseTests
18+
include(joinpath(dirname(dirname(@__DIR__)), "test", "choosetests.jl"))
19+
end
20+
21+
const tests = ChooseTests.choosetests() |>
22+
first |>
23+
x -> setdiff(x, exclude_tests) |>
24+
x -> vcat(x, include_tests) |>
25+
unique |>
26+
sort
27+
28+
const ncores = min(Sys.CPU_THREADS, Threads.nthreads())
29+
30+
@info "" ncores Sys.CPU_THREADS Threads.nthreads()
31+
32+
try
33+
Base.runtests(tests; ncores)
34+
catch ex
35+
@error "" exception=(ex, catch_backtrace())
36+
end

‎.buildkite/coverage-linux64/upload_coverage.jl

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
empty!(Base.DEPOT_PATH)
2+
push!(Base.DEPOT_PATH, mktempdir(; cleanup = true))
3+
4+
import Pkg
5+
import Logging
6+
import TOML
7+
8+
Pkg.add(; name = "Coverage", uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037", version = "1")
9+
Pkg.precompile()
10+
11+
import Coverage
12+
13+
function get_external_stdlib_names(stdlib_dir::AbstractString)
14+
filename_list = filter(x -> isfile(joinpath(stdlib_dir, x)), readdir(stdlib_dir))
15+
# find all of the files like `Pkg.version`, `Statistics.version`, etc.
16+
regex_matches_or_nothing = match.(Ref(r"^([\w].*?)\.version$"), filename_list)
17+
regex_matches = filter(x -> x !== nothing, regex_matches_or_nothing)
18+
# get the names of the external stdlibs, like `Pkg`, `Statistics`, etc.
19+
external_stdlib_names = only.(regex_matches)
20+
unique!(external_stdlib_names)
21+
sort!(external_stdlib_names)
22+
@info "# Begin list of external stdlibs"
23+
for (i, x) in enumerate(external_stdlib_names)
24+
@info "$(i). $(x)"
25+
end
26+
@info "# End list of external stdlibs"
27+
return external_stdlib_names
28+
end
29+
30+
function get_external_stdlib_prefixes(stdlib_dir::AbstractString)
31+
external_stdlib_names = get_external_stdlib_names(stdlib_dir)
32+
prefixes_1 = joinpath.(Ref(stdlib_dir), external_stdlib_names, Ref(""))
33+
prefixes_2 = joinpath.(Ref(stdlib_dir), string.(external_stdlib_names, Ref("-")))
34+
prefixes = vcat(prefixes_1, prefixes_2)
35+
unique!(prefixes)
36+
sort!(prefixes)
37+
# example of what `prefixes` might look like:
38+
# 4-element Vector{String}:
39+
# "stdlib/Pkg-"
40+
# "stdlib/Pkg/"
41+
# "stdlib/Statistics-"
42+
# "stdlib/Statistics/"
43+
return prefixes
44+
end
45+
46+
function print_coverage_summary(fc::Coverage.FileCoverage)
47+
cov_lines, tot_lines = Coverage.get_summary(fc)
48+
if cov_lines == tot_lines == 0
49+
cov_pct = 0
50+
else
51+
cov_pct = floor(Int, cov_lines/tot_lines * 100)
52+
end
53+
pad_1 = 71
54+
pad_2 = 15
55+
pad_3 = 15
56+
col_1 = rpad(fc.filename, pad_1)
57+
col_2 = rpad(string(cov_pct, " %"), pad_2)
58+
col_3 = string(
59+
rpad(string(cov_lines), pad_3),
60+
string(tot_lines),
61+
)
62+
@info "$(col_1) $(col_2) $(col_3)"
63+
return nothing
64+
end
65+
66+
function print_coverage_summary(
67+
fcs::Vector{Coverage.FileCoverage}, description::AbstractString,
68+
)
69+
cov_lines, tot_lines = Coverage.get_summary(fcs)
70+
if cov_lines == tot_lines == 0
71+
cov_pct = 0
72+
else
73+
cov_pct = floor(Int, cov_lines/tot_lines * 100)
74+
end
75+
@info "$(description): $(cov_pct)% ($(cov_lines)/$(tot_lines))"
76+
return nothing
77+
end
78+
79+
# `Coverage.process_folder` will have a LOT of `@info` statements that will make the log
80+
# way too long. So before we run `Coverage.process_folder`, we disable logging for `@info`
81+
# statements. After we run `Coverage.process_folder`, we re-enable logging for `@info`
82+
# statements.
83+
Logging.disable_logging(Logging.Info)
84+
const fcs = Coverage.merge_coverage_counts(
85+
Coverage.process_folder("base"),
86+
Coverage.process_folder("stdlib"),
87+
);
88+
Logging.disable_logging(Logging.Debug)
89+
90+
# Only include source code files. Exclude test files, benchmarking files, etc.
91+
filter!(fcs) do fc
92+
occursin(r"^base\/", fc.filename) || occursin("/src/", fc.filename)
93+
end;
94+
95+
# Exclude all external stdlibs (stdlibs that live in external repos).
96+
const external_stdlib_prefixes = get_external_stdlib_prefixes("stdlib")
97+
filter!(fcs) do fc
98+
all(x -> !startswith(fc.filename, x), external_stdlib_prefixes)
99+
end;
100+
101+
# Exclude all stdlib JLLs (stdlibs of the form `stdlib/*_jll/`).
102+
filter!(fcs) do fc
103+
!occursin(r"^stdlib\/[A-Za-z0-9]*?_jll\/", fc.filename)
104+
end
105+
106+
sort!(fcs; by = fc -> fc.filename)
107+
108+
print_coverage_summary.(fcs);
109+
print_coverage_summary(fcs, "Total")
110+
111+
# In order to upload to Codecov, you need to have the `CODECOV_TOKEN` environment variable defined.
112+
Coverage.Codecov.submit_local(fcs)
113+
114+
# In order to upload to Coveralls, you need to have the `COVERALLS_TOKEN` environment variable defined.
115+
Coverage.Coveralls.submit_local(fcs)

0 commit comments

Comments
 (0)