Skip to content

Commit 80112b1

Browse files
author
MarcoFalke
committed
Merge bitcoin#15296: tests: Add script checking for deterministic line coverage in unit tests
4320623 tests: Add script checking for deterministic line coverage (practicalswift) Pull request description: Add script checking for deterministic line coverage in unit tests. Context: bitcoin#14343 ("coverage reports non-deterministic") When the coverage is deterministic this script can be invoked from Travis to guard against regressions, but left inactive for now. Output in case of determinism: ``` $ contrib/test_deterministic_coverage.sh 2 [2019-01-30 20:08:46] Measuring coverage, run #1 of 2 [2019-01-30 20:10:45] Measuring coverage, run #2 of 2 Coverage test passed: Deterministic coverage across 2 runs. ``` Output in case of non-determinism: ``` $ contrib/test_deterministic_coverage.sh 2 [2019-01-30 20:08:46] Measuring coverage, run #1 of 2 [2019-01-30 20:10:45] Measuring coverage, run #2 of 2 The line coverage is non-deterministic between runs. The test suite must be deterministic in the sense that the set of lines executed at least once must be identical between runs. This is a neccessary condition for meaningful coverage measuring. --- gcovr.run-1.txt 2019-01-30 23:14:07.419418694 +0100 +++ gcovr.run-2.txt 2019-01-30 23:15:57.998811282 +0100 @@ -471,7 +471,7 @@ test/crypto_tests.cpp 270 270 100% test/cuckoocache_tests.cpp 142 142 100% test/dbwrapper_tests.cpp 148 148 100% -test/denialofservice_tests.cpp 225 225 100% +test/denialofservice_tests.cpp 225 224 99% 363 test/descriptor_tests.cpp 116 116 100% test/fs_tests.cpp 24 3 12% 14,16-17,19-20,23,25-26,29,31-32,35-36,39,41-42,45-46,49,51-52 test/getarg_tests.cpp 111 111 100% @@ -585,5 +585,5 @@ zmq/zmqpublishnotifier.h 5 0 0% 12,31,37,43,49 zmq/zmqrpc.cpp 21 0 0% 16,18,20,22,33-35,38-45,49,52,56,60,62-63 ------------------------------------------------------------------------------ -TOTAL 61561 27606 44% +TOTAL 61561 27605 44% ------------------------------------------------------------------------------ ``` In this case line 363 of `test/denialofservice_tests.cpp` was executed only in the second run. Non-determinism detected! Tree-SHA512: 03f45590e70a87146f89aa7838beeff0925d7fd303697ff03e0e69f8a5861694be5f0dd10cb0020e3e3d40c9cf662f71dfcd838f6affb31bd5212314e0a4e3a9
2 parents 9e3122d + 4320623 commit 80112b1

File tree

1 file changed

+151
-0
lines changed

1 file changed

+151
-0
lines changed

‎contrib/devtools/test_deterministic_coverage.sh‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
#!/usr/bin/env bash
2+
#
3+
# Copyright (c) 2019 The Bitcoin Core developers
4+
# Distributed under the MIT software license, see the accompanying
5+
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
6+
#
7+
# Test for deterministic coverage across unit test runs.
8+
9+
export LC_ALL=C
10+
11+
# Use GCOV_EXECUTABLE="gcov" if compiling with gcc.
12+
# Use GCOV_EXECUTABLE="llvm-cov gcov" if compiling with clang.
13+
GCOV_EXECUTABLE="gcov"
14+
15+
# Disable tests known to cause non-deterministic behaviour and document the source or point of non-determinism.
16+
NON_DETERMINISTIC_TESTS=(
17+
"coinselector_tests/knapsack_solver_test" # coinselector_tests.cpp: if (equal_sets(setCoinsRet, setCoinsRet2))
18+
"denialofservice_tests/DoS_mapOrphans" # denialofservice_tests.cpp: it = mapOrphanTransactions.lower_bound(InsecureRand256());
19+
"fs_tests/fsbridge_fstream" # deterministic test failure?
20+
"miner_tests/CreateNewBlock_validity" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
21+
"scheduler_tests/manythreads" # scheduler.cpp: CScheduler::serviceQueue()
22+
"scheduler_tests/singlethreadedscheduler_ordered" # scheduler.cpp: CScheduler::serviceQueue()
23+
"tx_validationcache_tests/checkinputs_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
24+
"tx_validationcache_tests/tx_mempool_block_doublespend" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
25+
"txindex_tests/txindex_initial_sync" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
26+
"txvalidation_tests/tx_mempool_reject_coinbase" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
27+
"validation_block_tests/processnewblock_signals_ordering" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
28+
"wallet_tests/coin_mark_dirty_immature_credit" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
29+
"wallet_tests/dummy_input_size_test" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
30+
"wallet_tests/importmulti_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
31+
"wallet_tests/importwallet_rescan" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
32+
"wallet_tests/ListCoins" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
33+
"wallet_tests/scan_for_wallet_transactions" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
34+
"wallet_tests/wallet_disableprivkeys" # validation.cpp: if (GetMainSignals().CallbacksPending() > 10)
35+
)
36+
37+
TEST_BITCOIN_BINARY="src/test/test_bitcoin"
38+
39+
print_usage() {
40+
echo "Usage: $0 [custom test filter (default: all but known non-deterministic tests)] [number of test runs (default: 2)]"
41+
}
42+
43+
N_TEST_RUNS=2
44+
BOOST_TEST_RUN_FILTERS=""
45+
if [[ $# != 0 ]]; then
46+
if [[ $1 == "--help" ]]; then
47+
print_usage
48+
exit
49+
fi
50+
PARSED_ARGUMENTS=0
51+
if [[ $1 =~ [a-z] ]]; then
52+
BOOST_TEST_RUN_FILTERS=$1
53+
PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
54+
shift
55+
fi
56+
if [[ $1 =~ ^[0-9]+$ ]]; then
57+
N_TEST_RUNS=$1
58+
PARSED_ARGUMENTS=$((PARSED_ARGUMENTS + 1))
59+
shift
60+
fi
61+
if [[ ${PARSED_ARGUMENTS} == 0 || $# -gt 2 || ${N_TEST_RUNS} -lt 2 ]]; then
62+
print_usage
63+
exit
64+
fi
65+
fi
66+
if [[ ${BOOST_TEST_RUN_FILTERS} == "" ]]; then
67+
BOOST_TEST_RUN_FILTERS="$(IFS=":"; echo "!${NON_DETERMINISTIC_TESTS[*]}" | sed 's/:/:!/g')"
68+
else
69+
echo "Using Boost test filter: ${BOOST_TEST_RUN_FILTERS}"
70+
echo
71+
fi
72+
73+
if ! command -v gcov > /dev/null; then
74+
echo "Error: gcov not installed. Exiting."
75+
exit 1
76+
fi
77+
78+
if ! command -v gcovr > /dev/null; then
79+
echo "Error: gcovr not installed. Exiting."
80+
exit 1
81+
fi
82+
83+
if [[ ! -e ${TEST_BITCOIN_BINARY} ]]; then
84+
echo "Error: Executable ${TEST_BITCOIN_BINARY} not found. Run \"./configure --enable-lcov\" and compile."
85+
exit 1
86+
fi
87+
88+
get_file_suffix_count() {
89+
find src/ -type f -name "*.$1" | wc -l
90+
}
91+
92+
if [[ $(get_file_suffix_count gcno) == 0 ]]; then
93+
echo "Error: Could not find any *.gcno files. The *.gcno files are generated by the compiler. Run \"./configure --enable-lcov\" and re-compile."
94+
exit 1
95+
fi
96+
97+
get_covr_filename() {
98+
echo "gcovr.run-$1.txt"
99+
}
100+
101+
TEST_RUN_ID=0
102+
while [[ ${TEST_RUN_ID} -lt ${N_TEST_RUNS} ]]; do
103+
TEST_RUN_ID=$((TEST_RUN_ID + 1))
104+
echo "[$(date +"%Y-%m-%d %H:%M:%S")] Measuring coverage, run #${TEST_RUN_ID} of ${N_TEST_RUNS}"
105+
find src/ -type f -name "*.gcda" -exec rm {} \;
106+
if [[ $(get_file_suffix_count gcda) != 0 ]]; then
107+
echo "Error: Stale *.gcda files found. Exiting."
108+
exit 1
109+
fi
110+
TEST_OUTPUT_TEMPFILE=$(mktemp)
111+
if ! BOOST_TEST_RUN_FILTERS="${BOOST_TEST_RUN_FILTERS}" ${TEST_BITCOIN_BINARY} > "${TEST_OUTPUT_TEMPFILE}" 2>&1; then
112+
cat "${TEST_OUTPUT_TEMPFILE}"
113+
rm "${TEST_OUTPUT_TEMPFILE}"
114+
exit 1
115+
fi
116+
rm "${TEST_OUTPUT_TEMPFILE}"
117+
if [[ $(get_file_suffix_count gcda) == 0 ]]; then
118+
echo "Error: Running the test suite did not create any *.gcda files. The gcda files are generated when the instrumented test programs are executed. Run \"./configure --enable-lcov\" and re-compile."
119+
exit 1
120+
fi
121+
GCOVR_TEMPFILE=$(mktemp)
122+
if ! gcovr --gcov-executable "${GCOV_EXECUTABLE}" -r src/ > "${GCOVR_TEMPFILE}"; then
123+
echo "Error: gcovr failed. Output written to ${GCOVR_TEMPFILE}. Exiting."
124+
exit 1
125+
fi
126+
GCOVR_FILENAME=$(get_covr_filename ${TEST_RUN_ID})
127+
mv "${GCOVR_TEMPFILE}" "${GCOVR_FILENAME}"
128+
if grep -E "^TOTAL *0 *0 " "${GCOVR_FILENAME}"; then
129+
echo "Error: Spurious gcovr output. Make sure the correct GCOV_EXECUTABLE variable is set in $0 (\"gcov\" for gcc, \"llvm-cov gcov\" for clang)."
130+
exit 1
131+
fi
132+
if [[ ${TEST_RUN_ID} != 1 ]]; then
133+
COVERAGE_DIFF=$(diff -u "$(get_covr_filename 1)" "${GCOVR_FILENAME}")
134+
if [[ ${COVERAGE_DIFF} != "" ]]; then
135+
echo
136+
echo "The line coverage is non-deterministic between runs. Exiting."
137+
echo
138+
echo "The test suite must be deterministic in the sense that the set of lines executed at least"
139+
echo "once must be identical between runs. This is a necessary condition for meaningful"
140+
echo "coverage measuring."
141+
echo
142+
echo "${COVERAGE_DIFF}"
143+
exit 1
144+
fi
145+
rm "${GCOVR_FILENAME}"
146+
fi
147+
done
148+
149+
echo
150+
echo "Coverage test passed: Deterministic coverage across ${N_TEST_RUNS} runs."
151+
exit

0 commit comments

Comments
 (0)