Merge bitcoin#17517: ci: Bump to clang-8 for asan build to avoid segfaults on ppc64le

fa40e48 ci: Remove unparseable lines from supp file for old xenial clang tsan (MarcoFalke)
fa1bfc4 ci: ubsan report_error_type=1 and add suppressions (MarcoFalke)
fa69cef test: Print stderr when subprocess fails (MarcoFalke)
2222c30 test: Use char instead of unsigned char (MarcoFalke)
faa8023 ci: Bump to clang-8 for asan build to avoid segfaults on ppc64le (MarcoFalke)

Pull request description:

  Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running the ci system on ppc64le

ACKs for top commit:
  practicalswift:
    ACK fa40e48 assuming Travis is happy -- diff looks correct :)

Tree-SHA512: f4f26232d3a0ef38da245869340f723d279a3db9823befbc735fb5a00024dae041c7306d7ae55d2488e6f86aa96cdea155b007aefb561fba505141e8dbc717dc

Author: MarcoFalke

ci/test/00_setup_env.sh

@@ -6,19 +6,19 @@
 
 export LC_ALL=C.UTF-8
 
+# The root dir.
+# The ci system copies this folder.
+# This is where the build is done (depends and dist).
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
 echo "Setting specific values in env"
 if [ -n "${FILE_ENV}" ]; then
   set -o errexit;
   # shellcheck disable=SC1090
   source "${FILE_ENV}"
 fi
 
-# The root dir.
-# The ci system copies this folder.
-# This is where the build is done (depends and dist).
-BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
-export BASE_ROOT_DIR
-
 echo "Fallback to default values in env (if not yet set)"
 # The number of parallel jobs to pass down to make and test_runner.py
 export MAKEJOBS=${MAKEJOBS:--j4}

ci/test/00_setup_env_native_asan.sh

@@ -6,7 +6,8 @@
 
 export LC_ALL=C.UTF-8
 
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+export PACKAGES="clang-8 llvm-8 python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+# Use clang-8 instead of default clang (which is clang-6 on Bionic) to avoid spurious segfaults when running on ppc64le
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang-8 CXX=clang++-8"

ci/test/00_setup_env_native_tsan.sh

@@ -11,3 +11,7 @@ export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-
 export NO_DEPENDS=1
 export GOAL="install"
 export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++"
+
+# xenial comes with old clang versions that can not parse the sanitizer suppressions files
+# Remove unparseable lines as a hacky workaround
+sed -i '/^implicit-/d' "${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan"

ci/test/04_install.sh

@@ -36,7 +36,7 @@ mkdir -p "${CCACHE_DIR}"
 export ASAN_OPTIONS="detect_stack_use_after_return=1"
 export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan"
-export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
+export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
 env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_ADMIN="--cap-add SYS_ADMIN"

src/test/crypto_tests.cpp

@@ -186,14 +186,15 @@ static void TestHKDF_SHA256_32(const std::string &ikm_hex, const std::string &sa
     BOOST_CHECK(HexStr(out, out + 32) == okm_check_hex);
 }
 
-static std::string LongTestString() {
+static std::string LongTestString()
+{
     std::string ret;
-    for (int i=0; i<200000; i++) {
-        ret += (unsigned char)(i);
-        ret += (unsigned char)(i >> 4);
-        ret += (unsigned char)(i >> 8);
-        ret += (unsigned char)(i >> 12);
-        ret += (unsigned char)(i >> 16);
+    for (int i = 0; i < 200000; i++) {
+        ret += (char)(i);
+        ret += (char)(i >> 4);
+        ret += (char)(i >> 8);
+        ret += (char)(i >> 12);
+        ret += (char)(i >> 16);
     }
     return ret;
 }

test/functional/test_framework/test_framework.py

@@ -12,6 +12,7 @@
 import pdb
 import random
 import shutil
+import subprocess
 import sys
 import tempfile
 import time
@@ -121,6 +122,9 @@ def main(self):
         except KeyError:
             self.log.exception("Key error")
             self.success = TestStatus.FAILED
+        except subprocess.CalledProcessError as e:
+            self.log.exception("Called Process failed with '{}'".format(e.output))
+            self.success = TestStatus.FAILED
         except Exception:
             self.log.exception("Unexpected exception caught during testing")
             self.success = TestStatus.FAILED

test/sanitizer_suppressions/ubsan

@@ -42,3 +42,45 @@ unsigned-integer-overflow:stl_bvector.h
 unsigned-integer-overflow:txmempool.cpp
 unsigned-integer-overflow:util/strencodings.cpp
 unsigned-integer-overflow:validation.cpp
+
+implicit-integer-sign-change:*/include/c++/*/bits/*.h
+implicit-integer-sign-change:*/new_allocator.h
+implicit-integer-sign-change:/usr/include/boost/date_time/format_date_parser.hpp
+implicit-integer-sign-change:arith_uint256.cpp
+implicit-integer-sign-change:bech32.cpp
+implicit-integer-sign-change:bloom.cpp
+implicit-integer-sign-change:chain.*
+implicit-integer-sign-change:coins.h
+implicit-integer-sign-change:compat/stdin.cpp
+implicit-integer-sign-change:compressor.h
+implicit-integer-sign-change:crypto/*
+implicit-integer-sign-change:key.cpp
+implicit-integer-sign-change:noui.cpp
+implicit-integer-sign-change:prevector.h
+implicit-integer-sign-change:protocol.cpp
+implicit-integer-sign-change:script/bitcoinconsensus.cpp
+implicit-integer-sign-change:script/interpreter.cpp
+implicit-integer-sign-change:serialize.h
+implicit-integer-sign-change:test/arith_uint256_tests.cpp
+implicit-integer-sign-change:test/coins_tests.cpp
+implicit-integer-sign-change:test/pow_tests.cpp
+implicit-integer-sign-change:test/prevector_tests.cpp
+implicit-integer-sign-change:test/sighash_tests.cpp
+implicit-integer-sign-change:test/streams_tests.cpp
+implicit-integer-sign-change:test/transaction_tests.cpp
+implicit-integer-sign-change:txmempool.cpp
+implicit-integer-sign-change:util/strencodings.*
+implicit-integer-sign-change:validation.cpp
+implicit-integer-sign-change:zmq/zmqpublishnotifier.cpp
+implicit-signed-integer-truncation,implicit-integer-sign-change:chain.h
+implicit-signed-integer-truncation,implicit-integer-sign-change:test/skiplist_tests.cpp
+implicit-signed-integer-truncation:chain.h
+implicit-signed-integer-truncation:crypto/*
+implicit-signed-integer-truncation:cuckoocache.h
+implicit-signed-integer-truncation:leveldb/*
+implicit-signed-integer-truncation:streams.h
+implicit-signed-integer-truncation:test/arith_uint256_tests.cpp
+implicit-signed-integer-truncation:test/skiplist_tests.cpp
+implicit-signed-integer-truncation:torcontrol.cpp
+implicit-unsigned-integer-truncation:crypto/*
+implicit-unsigned-integer-truncation:leveldb/*