Skip to content

Commit 2b6db2b

Browse files
eddyz87eddyz87
committed
add veristat test.yml job
1 parent b719a06 commit 2b6db2b

File tree

3 files changed

+267
-7
lines changed

3 files changed

+267
-7
lines changed

‎.github/scripts/veristat-compare.py‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
import sys
2+
import csv
3+
import os
4+
5+
# File format:
6+
#
7+
# Columns:
8+
# 0. file_name
9+
# 1. prog_name
10+
# 2. verdict_base
11+
# 3. verdict_comp
12+
# 4. verdict_diff
13+
# 5. total_states_base
14+
# 6. total_states_comp
15+
# 7. total_states_diff
16+
#
17+
# Records sample:
18+
# file-a,a,success,failure,MISMATCH,12,12,+0 (+0.00%)
19+
# file-b,b,success,success,MATCH,67,67,+0 (+0.00%)
20+
21+
TRESHOLD_PCT = 0
22+
23+
HEADERS = ['file_name', 'prog_name', 'verdict_base', 'verdict_comp',
24+
'verdict_diff', 'total_states_base', 'total_states_comp',
25+
'total_states_diff']
26+
27+
FILE = 0
28+
PROG = 1
29+
VERDICT_OLD = 2
30+
VERDICT_NEW = 3
31+
STATES_OLD = 5
32+
STATES_NEW = 6
33+
34+
if len(sys.argv) == 2:
35+
CSV_PATH = sys.argv[1]
36+
else:
37+
print(f'Usage: {sys.argv[0]} <compare.csv>>')
38+
sys.exit(1)
39+
40+
if not os.path.exists(CSV_PATH):
41+
print(f'# {CSV_PATH} does not exist, failing step')
42+
sys.exit(1)
43+
44+
def read_table(path):
45+
with open(path, newline='') as file:
46+
reader = csv.reader(file)
47+
headers = next(reader)
48+
if headers != HEADERS:
49+
raise Exception(f'Unexpected table header for {path}: {headers}')
50+
return list(reader)
51+
52+
table = read_table(CSV_PATH)
53+
new_failures = False
54+
changes = False
55+
headers = ['File', 'Program', 'Verdict', 'States Diff (%)']
56+
html_table = [headers]
57+
text_table = [headers]
58+
59+
def compute_diff(v):
60+
old = int(v[STATES_OLD]) if v[STATES_OLD] != 'N/A' else 0
61+
new = int(v[STATES_NEW]) if v[STATES_NEW] != 'N/A' else 0
62+
if old == 0:
63+
return 1
64+
return (new - old) / old
65+
66+
for v in table:
67+
add = False
68+
html_verdict = v[VERDICT_NEW]
69+
text_verdict = v[VERDICT_NEW]
70+
diff = compute_diff(v)
71+
72+
if v[VERDICT_OLD] != v[VERDICT_NEW]:
73+
changes = True
74+
add = True
75+
html_verdict = f'{v[VERDICT_OLD]} &rarr; {v[VERDICT_NEW]}'
76+
text_verdict = f'{v[VERDICT_OLD]} -> {v[VERDICT_NEW]}'
77+
if v[VERDICT_NEW] == 'failure':
78+
new_failures = True
79+
html_verdict += ' :bangbang:'
80+
text_verdict += ' (!!)'
81+
82+
if abs(diff * 100) > TRESHOLD_PCT:
83+
changes = True
84+
add = True
85+
86+
if not add:
87+
continue
88+
89+
diff_txt = '{:+.1f} %'.format(diff * 100)
90+
html_table.append([v[FILE], v[PROG], html_verdict, diff_txt])
91+
text_table.append([v[FILE], v[PROG], text_verdict, diff_txt])
92+
93+
def print_table(rows, fn):
94+
column_widths = [0] * len(rows[0])
95+
for row in rows:
96+
for i, col in enumerate(row):
97+
column_widths[i] = max(column_widths[i], len(col))
98+
99+
def print_row(row):
100+
line = '|'
101+
for width, col in zip(column_widths, row):
102+
line += ' ' + col.ljust(width) + ' |'
103+
fn(line)
104+
105+
rows_iter = iter(rows)
106+
print_row(next(rows_iter))
107+
108+
underscores = '|'
109+
for width in column_widths:
110+
underscores += '-' * (width + 2) + '|'
111+
fn(underscores)
112+
113+
for row in rows_iter:
114+
print_row(row)
115+
116+
if new_failures:
117+
section_name = '# There are new veristat failures'
118+
elif changes:
119+
section_name = '# There are changes in verification performance'
120+
else:
121+
section_name = '# No changes in verification performance'
122+
123+
# Print the step summary
124+
125+
if summary_path := os.getenv('GITHUB_STEP_SUMMARY', None):
126+
summary_file = open(summary_path, 'a')
127+
128+
def print_summary(s = ''):
129+
if summary_file:
130+
summary_file.write(f'{s}\n')
131+
132+
print_summary(section_name)
133+
print(section_name)
134+
135+
if new_failures or changes:
136+
print()
137+
print_table(text_table, print)
138+
print()
139+
140+
print_summary()
141+
print_summary('<details>')
142+
print_summary('<summary>Click to expand</summary>')
143+
print_summary()
144+
print_table(html_table, print_summary)
145+
print_summary()
146+
print_summary('</details>')
147+
148+
if new_failures:
149+
print('Failing step because of the new failures')
150+
sys.exit(1)

‎.github/workflows/test.yml‎

Lines changed: 102 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,12 @@ on:
66
branches:
77
- bpf_base
88
- bpf-next_base
9+
- bpf-next
910

11+
env:
12+
veristat_arch: x86_64
13+
veristat_toolchain: gcc
14+
1015
concurrency:
1116
group: ci-test-${{ github.ref_name }}
1217
cancel-in-progress: true
@@ -17,6 +22,7 @@ jobs:
1722
outputs:
1823
build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }}
1924
test-matrix: ${{ steps.set-matrix-impl.outputs.test_matrix }}
25+
veristat-runs-on: ${{ steps.set-matrix-impl.outputs.veristat_runs_on }}
2026
steps:
2127
- id: set-matrix-impl
2228
shell: python3 -I {0}
@@ -109,6 +115,12 @@ jobs:
109115
for test in get_tests(config)
110116
]}
111117
set_output("test_matrix", dumps(test_matrix))
118+
119+
veristat_runs_on = next(x['runs_on']
120+
for x in matrix
121+
if x['arch'] == "${{env.veristat_arch}}" and
122+
x['toolchain'] == "${{env.veristat_toolchain}}")
123+
set_output("veristat_runs_on", veristat_runs_on)
112124
build:
113125
name: build for ${{ matrix.arch }} with ${{ matrix.toolchain_full }}
114126
needs: set-matrix
@@ -243,9 +255,8 @@ jobs:
243255
kbuild-output: ${{ env.KBUILD_OUTPUT }}
244256
max-make-jobs: 32
245257
llvm-version: ${{ matrix.llvm-version }}
246-
- if: ${{ github.event_name != 'push' }}
247-
name: Build selftests
248-
uses: libbpf/ci/build-selftests@main
258+
- name: Build selftests
259+
uses: eddyz87/libbpf-ci/build-selftests@main
249260
with:
250261
toolchain: ${{ matrix.toolchain }}
251262
kbuild-output: ${{ env.KBUILD_OUTPUT }}
@@ -259,8 +270,7 @@ jobs:
259270
kbuild-output: ${{ env.KBUILD_OUTPUT }}
260271
max-make-jobs: 32
261272
llvm-version: ${{ matrix.llvm-version }}
262-
- if: ${{ github.event_name != 'push' }}
263-
name: Tar artifacts
273+
- name: Tar artifacts
264274
run: |
265275
# Remove intermediate object files that we have no use for. Ideally
266276
# we'd just exclude them from tar below, but it does not provide
@@ -304,8 +314,7 @@ jobs:
304314
# Only on pushed changes are build artifacts actually cached, because
305315
# of github.com/actions/cache's cache isolation logic.
306316
rm -rf "${KBUILD_OUTPUT}"
307-
- if: ${{ github.event_name != 'push' }}
308-
uses: actions/upload-artifact@v3
317+
- uses: actions/upload-artifact@v3
309318
with:
310319
name: vmlinux-${{ matrix.arch }}-${{ matrix.toolchain_full }}
311320
if-no-files-found: error
@@ -354,3 +363,89 @@ jobs:
354363
kernel-root: '.'
355364
max-cpu: 8
356365
kernel-test: ${{ matrix.test }}
366+
veristat:
367+
name: veristat
368+
needs: [set-matrix, build]
369+
runs-on: ${{ fromJSON(needs.set-matrix.outputs.veristat-runs-on) }}
370+
timeout-minutes: 100
371+
env:
372+
KERNEL: LATEST
373+
REPO_ROOT: ${{ github.workspace }}
374+
REPO_PATH: ""
375+
KBUILD_OUTPUT: kbuild-output/
376+
steps:
377+
- name: Setup environment variables
378+
run: |
379+
echo arch_and_tool=${{ env.veristat_arch }}-${{ env.veristat_toolchain }} > \
380+
${GITHUB_ENV}
381+
- uses: actions/checkout@v3
382+
- uses: actions/download-artifact@v3
383+
with:
384+
name: vmlinux-${{ env.arch_and_tool }}
385+
path: .
386+
- name: Untar artifacts
387+
# zstd is installed by default in the runner images.
388+
run: zstd -d -T0 vmlinux-${{ env.arch_and_tool }}.tar.zst --stdout | tar -xf -
389+
390+
- name: Prepare rootfs
391+
uses: eddyz87/libbpf-ci/prepare-rootfs@main
392+
with:
393+
project-name: 'libbpf'
394+
arch: x86_64
395+
kernel: LATEST
396+
kernel-root: '.'
397+
kbuild-output: ${{ env.KBUILD_OUTPUT }}
398+
image-output: '/tmp/root.img'
399+
test: run_veristat
400+
401+
- name: Run veristat
402+
uses: eddyz87/libbpf-ci/run-qemu@main
403+
timeout-minutes: 10
404+
with:
405+
arch: x86_64
406+
img: '/tmp/root.img'
407+
vmlinuz: '${{ github.workspace }}/vmlinuz'
408+
kernel-root: '.'
409+
max-cpu: 8
410+
kernel-test: run_veristat
411+
output-dir: '${{ github.workspace }}'
412+
413+
# veristat.csv is produced by run-qemu run_veristat action
414+
- uses: actions/upload-artifact@v3
415+
with:
416+
name: ${{ env.arch_and_tool }}-veristat-log
417+
if-no-files-found: error
418+
path: '${{ github.workspace }}/veristat.csv'
419+
420+
# For pull request:
421+
# - get baseline log from cache
422+
# - compare it to current run
423+
- if: ${{ github.event_name == 'pull_request' }}
424+
uses: actions/cache/restore@v3
425+
with:
426+
key: ${{ env.arch_and_tool }}-veristat-baseline
427+
restore-keys: |
428+
${{ env.arch_and_tool }}-veristat-baseline-
429+
path: '${{ github.workspace }}/veristat-baseline.csv'
430+
431+
- if: ${{ github.event_name == 'pull_request' }}
432+
name: Show veristat comparison
433+
run: |
434+
cd ${{ github.workspace }}
435+
selftests/bpf/veristat \
436+
--output-format csv \
437+
--emit file,prog,verdict,states \
438+
--compare veristat-baseline.csv veristat.csv > compare.csv
439+
python3 ./.github/scripts/veristat-compare.py compare.csv
440+
441+
# For push: just put baseline log to cache
442+
- if: ${{ github.event_name == 'push' }}
443+
run: |
444+
mv '${{ github.workspace }}/veristat.csv' \
445+
'${{ github.workspace }}/veristat-baseline.csv'
446+
447+
- if: ${{ github.event_name == 'push' }}
448+
uses: actions/cache/save@v3
449+
with:
450+
key: ${{ env.arch_and_tool }}-veristat-baseline-${{ github.run_id }}
451+
path: '${{ github.workspace }}/veristat-baseline.csv'

‎ci/vmtest/run_selftests.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ source "$(cd "$(dirname "$0")" && pwd)/helpers.sh"
1515
ARCH=$(uname -m)
1616

1717
STATUS_FILE=/exitstatus
18+
OUTPUT_DIR=/command_output
1819

1920
declare -a TEST_NAMES=()
2021

@@ -98,6 +99,20 @@ test_verifier() {
9899
foldable end test_verifier
99100
}
100101
102+
run_veristat() {
103+
foldable start run_veristat "Running veristat"
104+
105+
globs=$(awk '/^#/ { next; } { print $0 ".bpf.o"; }' ./veristat.cfg)
106+
mkdir -p ${OUTPUT_DIR}
107+
./veristat --outpot-format csv \
108+
--quiet \
109+
--emit file,prog,verdict,states \
110+
${globs} > ${OUTPUT_DIR}/veristat.csv
111+
echo "run_veristat:$?" >> ${STATUS_FILE}
112+
113+
foldable end run_veristat
114+
}
115+
101116
foldable end vm_init
102117
103118
foldable start kernel_config "Kconfig"

0 commit comments

Comments
 (0)