Merge pull request dashpay#5340 from knst/bc-bp-v20-missing-4

backport: bitcoin#16658, bitcoin#17473, bitcoin#17549, bitcoin#17578, bitcoin#17593, bitcoin#17661, bitcoin#17674, bitcoin#17678, bitcoin#17725, bitcoin#17762, dashpay#1780

Author: UdjinM6

.travis.yml

@@ -212,19 +212,6 @@ after_success:
       script:
         - set -o errexit; source ./ci/lint/06_script.sh
 
-    - stage: extended-lint
-      name: 'extended lint [runtime >= 60 seconds]'
-      env:
-      cache: false
-      language: python
-      python: '3.5'
-      install:
-        - set -o errexit; source ./ci/extended_lint/04_install.sh
-      before_script:
-        - set -o errexit; source ./ci/lint/05_before_script.sh
-      script:
-        - set -o errexit; source ./ci/extended_lint/06_script.sh
-
     - stage: test
       name: 'ARM  [GOAL: install]  [bionic]  [unit tests, functional tests]'
       arch: arm64
@@ -237,6 +224,7 @@ after_success:
       arch: s390x
       env: >-
         FILE_ENV="./ci/test/00_setup_env_s390x.sh"
+        QEMU_USER_CMD=""  # Can run the tests natively without qemu
 
     - stage: test
       name: 'Win64  [GOAL: deploy]  [unit tests, no gui, no functional tests]'

ci/dash/build-docker.sh

@@ -12,17 +12,14 @@ DOCKER_IMAGE=${DOCKER_IMAGE:-dashpay/dashd-develop}
 DOCKER_TAG=${DOCKER_TAG:-latest}
 DOCKER_RELATIVE_PATH=contrib/containers/deploy
 
-BASE_BUILD_DIR=${BASE_BUILD_DIR:-.}
-
-
 if [ -d $DOCKER_RELATIVE_PATH/bin ]; then
     rm $DOCKER_RELATIVE_PATH/bin/*
 fi
 
 mkdir $DOCKER_RELATIVE_PATH/bin
-cp "$BASE_BUILD_DIR"/src/dashd    $DOCKER_RELATIVE_PATH/bin/
-cp "$BASE_BUILD_DIR"/src/dash-cli $DOCKER_RELATIVE_PATH/bin/
-cp "$BASE_BUILD_DIR"/src/dash-tx  $DOCKER_RELATIVE_PATH/bin/
+cp "$BASE_ROOT_DIR"/src/dashd    $DOCKER_RELATIVE_PATH/bin/
+cp "$BASE_ROOT_DIR"/src/dash-cli $DOCKER_RELATIVE_PATH/bin/
+cp "$BASE_ROOT_DIR"/src/dash-tx  $DOCKER_RELATIVE_PATH/bin/
 strip $DOCKER_RELATIVE_PATH/bin/dashd
 strip $DOCKER_RELATIVE_PATH/bin/dash-cli
 strip $DOCKER_RELATIVE_PATH/bin/dash-tx

ci/dash/build_depends.sh

@@ -17,19 +17,19 @@ unset DISPLAY
 mkdir -p $CACHE_DIR/depends
 mkdir -p $CACHE_DIR/sdk-sources
 
-ln -s $CACHE_DIR/depends depends/built
-ln -s $CACHE_DIR/sdk-sources depends/sdk-sources
+ln -s $CACHE_DIR/depends ${DEPENDS_DIR}/built
+ln -s $CACHE_DIR/sdk-sources ${DEPENDS_DIR}/sdk-sources
 
-mkdir -p depends/SDKs
+mkdir -p ${DEPENDS_DIR}/SDKs
 
 if [ -n "$XCODE_VERSION" ]; then
   OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-libcxx-headers.tar.gz"
-  OSX_SDK_PATH="depends/sdk-sources/${OSX_SDK_BASENAME}"
+  OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}"
   if [ ! -f "$OSX_SDK_PATH" ]; then
     curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
   fi
   if [ -f "$OSX_SDK_PATH" ]; then
-    tar -C depends/SDKs -xf "$OSX_SDK_PATH"
+    tar -C ${DEPENDS_DIR}/SDKs -xf "$OSX_SDK_PATH"
   fi
 fi
 

ci/dash/build_src.sh

@@ -40,7 +40,7 @@ if [ -n "$CONFIG_SHELL" ]; then
   export CONFIG_SHELL="$CONFIG_SHELL"
 fi
 
-BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
+BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
 
 ( test -n "$CONFIG_SHELL" && eval '"$CONFIG_SHELL" -c "./autogen.sh"' ) || ./autogen.sh
 

ci/dash/test_integrationtests.sh

@@ -18,7 +18,7 @@ if [ "$RUN_INTEGRATION_TESTS" != "true" ]; then
   exit 0
 fi
 
-export LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib
+export LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib
 
 cd build-ci/dashcore-$BUILD_TARGET
 

ci/dash/test_unittests.sh

@@ -17,7 +17,7 @@ if [ "$RUN_UNIT_TESTS" != "true" ]; then
 fi
 
 export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1}
-export LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib
+export LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib
 
 export WINEDEBUG=fixme-all
 export BOOST_TEST_LOG_LEVEL=test_suite

ci/extended_lint/04_install.sh

@@ -28,6 +28,7 @@ echo "Fallback to default values in env (if not yet set)"
 MAKEJOBS="-j$(nproc)"
 export MAKEJOBS
 # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
+# This folder only exists on the ci host.
 export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
 # What host to compile for. See also ./depends/README.md
 # Tests that need cross-compilation export the appropriate HOST.
@@ -47,16 +48,19 @@ export CACHE_DIR=${CACHE_DIR:-$HOST_CACHE_DIR}
 export CCACHE_SIZE=${CCACHE_SIZE:-100M}
 export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
 export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+# The cache dir.
+# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
 export CCACHE_DIR=${CCACHE_DIR:-$CACHE_DIR/ccache}
-# Folder where the build is done (depends and dist). Can not be changed and is equal to the root of the git repo
-export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
-# Folder where the build is done (bin and lib). Can not be changed.
-export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out}
+# The depends dir.
+# This folder exists on the ci host and ci guest. Changes are propagated back and forth.
+export DEPENDS_DIR=${DEPENDS_DIR:-$BASE_ROOT_DIR/depends}
+# Folder where the build is done (bin and lib).
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export WINEDEBUG=${WINEDEBUG:-fixme-all}
 export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
 export GOAL=${GOAL:-install}
-export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
+export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
 export CI_RETRY_EXE=${CI_RETRY_EXE:-"retry --"}
 # Dash's Docker-specifics

ci/extended_lint/06_script.sh

@@ -6,9 +6,11 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=s390x-unknown-linux-gnu
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
-export NO_DEPENDS=1
+export HOST=s390x-linux-gnu
+# The host arch is unknown, so we run the tests through qemu.
+# If the host is s390x and wants to run the tests natively, it can set QEMU_USER_CMD to the empty string.
+export QEMU_USER_CMD="${QEMU_USER_CMD:"qemu-s390x"}"
+export PACKAGES="python3-zmq bsdmainutils qemu-user"
 export RUN_UNIT_TESTS=true
 export RUN_FUNCTIONAL_TESTS=true
 export GOAL="install"

ci/test/00_setup_env.sh

@@ -40,7 +40,7 @@ fi
 mkdir -p "${BASE_SCRATCH_DIR}"
 mkdir -p "${CCACHE_DIR}"
 
-export ASAN_OPTIONS="detect_stack_use_after_return=1"
+export ASAN_OPTIONS="detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"
 export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan"
 export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1"
@@ -51,25 +51,27 @@ elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (A
   DOCKER_ADMIN="--cap-add SYS_PTRACE"
 fi
 
+export P_CI_DIR="$PWD"
+
 if [ -z "$RUN_CI_ON_HOST" ]; then
   echo "Creating $DOCKER_NAME_TAG container to run in"
   ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
 
   DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
-                  --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \
+                  --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \
                   --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
-                  --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \
-                  -w $BASE_BUILD_DIR \
+                  --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \
+                  -w $BASE_ROOT_DIR \
                   --env-file /tmp/env \
                   $DOCKER_NAME_TAG)
 
   DOCKER_EXEC () {
-    docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
+    docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
   }
 else
   echo "Running on host system without docker wrapper"
   DOCKER_EXEC () {
-    bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
+    bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $P_CI_DIR && $*"
   }
 fi
 export -f DOCKER_EXEC
@@ -101,11 +103,11 @@ if [ ! -d ${DIR_QA_ASSETS} ]; then
 fi
 export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
 
-DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
+DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/"
 
 if [ -z "$RUN_CI_ON_HOST" ]; then
-  echo "Create $BASE_BUILD_DIR"
-  DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR
+  echo "Create $BASE_ROOT_DIR"
+  DOCKER_EXEC rsync -a /ro_base/ $BASE_ROOT_DIR
 fi
 
 if [ "$USE_BUSY_BOX" = "true" ]; then

ci/test/00_setup_env_s390x.sh

@@ -13,13 +13,13 @@ else
   DOCKER_EXEC echo \> \$HOME/.dashcore
 fi
 
-DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources
+DOCKER_EXEC mkdir -p ${DEPENDS_DIR}/SDKs ${DEPENDS_DIR}/sdk-sources
 
 if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then
   DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH"
 fi
 if [ -n "$XCODE_VERSION" ] && [ -f "$OSX_SDK_PATH" ]; then
-  DOCKER_EXEC tar -C "depends/SDKs" -xf "$OSX_SDK_PATH"
+  DOCKER_EXEC tar -C "${DEPENDS_DIR}/SDKs" -xf "$OSX_SDK_PATH"
 fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)

ci/test/04_install.sh

@@ -4,7 +4,7 @@ Upstream-Contact: Dash Core Group, Inc https://www.dash.org/team/
 Source: https://github.com/dashpay/dash
 
 Files: *
-Copyright: 2009-2019, Bitcoin Core Developers,
+Copyright: 2009-2020, Bitcoin Core Developers,
            2019-2020, Dash Core Developers
 License: Expat
 Comment: The Bitcoin Core Developers encompasses the current developers listed on bitcoin.org,

ci/test/05_before_script.sh

@@ -8,5 +8,7 @@ i686*
 mips*
 arm*
 aarch64*
+powerpc*
 riscv32*
 riscv64*
+s390x*

contrib/debian/copyright

@@ -245,7 +245,7 @@ $(host_prefix)/share/config.site: check-packages
 check-packages: check-sources
 
 clean-all: clean
-	@rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* riscv32* riscv64*
+	@rm -rf $(SOURCES_PATH) x86_64* i686* mips* arm* aarch64* powerpc* riscv32* riscv64* s390x*
 
 clean:
 	@rm -rf $(WORK_PATH) $(BASE_CACHE) $(BUILD)

depends/.gitignore

@@ -12,13 +12,13 @@ For example:
 
     make HOST=x86_64-w64-mingw32 -j4
 
-**Dash's configure script by default will ignore the depends output.** In
+**Dash Core's configure script by default will ignore the depends output.** In
 order for it to pick up libraries, tools, and settings from the depends build,
 you must point it at the appropriate `--prefix` directory generated by the
 build. In the above example, a prefix dir named x86_64-w64-mingw32 will be
 created. To use it for Dash:
 
-    ./configure --prefix=`pwd`/depends/x86_64-w64-mingw32
+    ./configure --prefix=$PWD/depends/x86_64-w64-mingw32
 
 Common `host-platform-triplets` for cross compilation are:
 
@@ -28,8 +28,11 @@ Common `host-platform-triplets` for cross compilation are:
 - `x86_64-apple-darwin19` for macOS
 - `arm-linux-gnueabihf` for Linux ARM 32 bit
 - `aarch64-linux-gnu` for Linux ARM 64 bit
+- `powerpc64-linux-gnu` for Linux POWER 64-bit (big endian)
+- `powerpc64le-linux-gnu` for Linux POWER 64-bit (little endian)
 - `riscv32-linux-gnu` for Linux RISC-V 32 bit
 - `riscv64-linux-gnu` for Linux RISC-V 64 bit
+- `s390x-linux-gnu` for Linux S390X
 - `armv7a-linux-android` for Android ARM 32 bit
 - `aarch64-linux-android` for Android ARM 64 bit
 - `i686-linux-android` for Android x86 32 bit
@@ -66,13 +69,21 @@ For linux AARCH64 cross compilation:
 
     sudo apt-get install g++-aarch64-linux-gnu binutils-aarch64-linux-gnu
 
+For linux POWER 64-bit cross compilation (there are no packages for 32-bit):
+
+    sudo apt-get install g++-powerpc64-linux-gnu binutils-powerpc64-linux-gnu g++-powerpc64le-linux-gnu binutils-powerpc64le-linux-gnu
+
 For linux RISC-V 64-bit cross compilation (there are no packages for 32-bit):
 
     sudo apt-get install g++-riscv64-linux-gnu binutils-riscv64-linux-gnu
 
 RISC-V known issue: gcc-7.3.0 and gcc-7.3.1 result in a broken `test_dash` executable (see https://github.com/bitcoin/bitcoin/pull/13543),
 this is apparently fixed in gcc-8.1.0.
 
+For linux S390X cross compilation:
+
+    sudo apt-get install g++-s390x-linux-gnu binutils-s390x-linux-gnu
+
 ### Dependency Options
 The following can be set when running make: `make FOO=bar`
 

depends/Makefile

@@ -151,7 +151,10 @@ else
 $(package)_config_opts_x86_64_linux = -xplatform linux-g++-64
 endif
 $(package)_config_opts_aarch64_linux = -xplatform linux-aarch64-gnu-g++
+$(package)_config_opts_powerpc64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++
+$(package)_config_opts_powerpc64le_linux = -platform linux-g++ -xplatform bitcoin-linux-g++
 $(package)_config_opts_riscv64_linux = -platform linux-g++ -xplatform bitcoin-linux-g++
+$(package)_config_opts_s390x_linux = -platform linux-g++ -xplatform bitcoin-linux-g++
 
 $(package)_config_opts_mingw32 = -no-opengl
 $(package)_config_opts_mingw32 += -no-dbus

depends/README.md

@@ -0,0 +1,8 @@
+Deprecated or removed RPCs
+--------------------------
+
+- The `getaddressinfo` RPC `labels` field now returns an array of label name
+  strings. Previously, it returned an array of JSON objects containing `name` and
+  `purpose` key/value pairs, which is now deprecated and will be removed in
+  0.21. To re-enable the previous behavior, launch bitcoind with
+  `-deprecatedrpc=labelspurpose`.

depends/packages/qt.mk

@@ -47,9 +47,7 @@ bench_bench_dash_SOURCES = \
   bench/lockedpool.cpp \
   bench/poly1305.cpp \
   bench/prevector.cpp \
-  bench/string_cast.cpp \
-  test/util.cpp \
-  test/util.h
+  bench/string_cast.cpp
 
 nodist_bench_bench_dash_SOURCES = $(GENERATED_BENCH_FILES)
 

doc/release-notes-17578.md

@@ -11,13 +11,13 @@ TEST_FUZZ_H = \
     test/fuzz/fuzz.h \
     test/fuzz/FuzzedDataProvider.h \
     test/fuzz/util.h \
-    test/util.h
+    test/util/mining.h
 
 libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS)
 libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libtest_fuzz_a_SOURCES = \
   test/fuzz/fuzz.cpp \
-  test/util.cpp \
+  test/util/mining.cpp \
   $(TEST_FUZZ_H)
 
 LIBTEST_FUZZ += $(LIBBITCOIN_SERVER)

src/Makefile.bench.include

@@ -10,24 +10,27 @@ EXTRA_LIBRARIES += \
 TEST_UTIL_H = \
     test/util/blockfilter.h \
     test/util/logging.h \
+    test/util/mining.h \
     test/util/net.h \
     test/util/setup_common.h \
     test/util/str.h \
-    test/util/transaction_utils.h
+    test/util/transaction_utils.h \
+    test/util/wallet.h
 
 libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS)
 libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 libtest_util_a_SOURCES = \
   test/util/blockfilter.cpp \
   test/util/logging.cpp \
+  test/util/mining.cpp \
   test/util/net.cpp \
   test/util/setup_common.cpp \
   test/util/str.cpp \
   test/util/transaction_utils.cpp \
+  test/util/wallet.cpp \
   $(TEST_UTIL_H)
 
 LIBTEST_UTIL += $(LIBBITCOIN_SERVER)
 LIBTEST_UTIL += $(LIBBITCOIN_COMMON)
 LIBTEST_UTIL += $(LIBBITCOIN_UTIL)
 LIBTEST_UTIL += $(LIBBITCOIN_CRYPTO_BASE)
-