rebroad
diff --git a/‎.cirrus.yml‎
Lines changed: 175 additions & 198 deletions b/‎.cirrus.yml‎
Lines changed: 175 additions & 198 deletions
@@ -1,221 +1,198 @@
-### Global defaults
-
 env:
-  PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
-  MAKEJOBS: "-j4"
-  TEST_RUNNER_PORT_MIN: "14000"  # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache
-  CCACHE_SIZE: "200M"
-  CCACHE_DIR: "/tmp/ccache_dir"
-  CCACHE_NOHASHDIR: "1"  # Debug info might contain a stale path if the build dir changes, but this is fine
-
-cirrus_ephemeral_worker_template_env: &CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-  DANGER_RUN_CI_ON_HOST: "1"  # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system
-
-persistent_worker_template_env: &PERSISTENT_WORKER_TEMPLATE_ENV
-  RESTART_CI_DOCKER_BEFORE_RUN: "1"
-
-persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE
-  persistent_worker: {}  # https://cirrus-ci.org/guide/persistent-workers/
-
-# https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
-base_template: &BASE_TEMPLATE
-  skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
+  WIDEMUL: auto
+  STATICPRECOMPUTATION: yes
+  ECMULTGENPRECISION: auto
+  ASM: no
+  BUILD: check
+  WITH_VALGRIND: yes
+  RUN_VALGRIND: no
+  EXTRAFLAGS:
+  HOST:
+  ECDH: no
+  RECOVERY: no
+  SCHNORRSIG: no
+  EXPERIMENTAL: no
+  CTIMETEST: yes
+  BENCH: yes
+  ITERS: 2
+  MAKEFLAGS: -j2
+
+cat_logs_snippet: &CAT_LOGS
+  always:
+    cat_tests_log_script:
+      - cat tests.log || true
+    cat_exhaustive_tests_log_script:
+      - cat exhaustive_tests.log || true
+    cat_valgrind_ctime_test_log_script:
+      - cat valgrind_ctime_test.log || true
+    cat_bench_log_script:
+      - cat bench.log || true
+  on_failure:
+    cat_config_log_script:
+      - cat config.log || true
+    cat_test_env_script:
+      - cat test_env.log || true
+    cat_ci_env_script:
+      - env
+
+merge_base_script_snippet: &MERGE_BASE
   merge_base_script:
     - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
-    - bash -c "$PACKAGE_MANAGER_INSTALL git"
     - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
     - git config --global user.email "ci@ci.ci"
     - git config --global user.name "ci"
     - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
-  stateful: false  # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks
-
-global_task_template: &GLOBAL_TASK_TEMPLATE
-  << : *BASE_TEMPLATE
-  timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
-  container:
-    # https://cirrus-ci.org/faq/#are-there-any-limits
-    # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
-    cpu: 2
-    memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
-  ccache_cache:
-    folder: "/tmp/ccache_dir"
-  depends_built_cache:
-    folder: "depends/built"
-  ci_script:
-    - ./ci/test_run_all.sh
-
-depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE
-  depends_sdk_cache:
-    folder: "depends/sdk-sources"
-
-compute_credits_template: &CREDITS_TEMPLATE
-  # https://cirrus-ci.org/pricing/#compute-credits
-  # Only use credits for pull requests to the main repo
-  use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'bitcoin/bitcoin' && $CIRRUS_PR != ""
-
-#task:
-#  name: "Windows"
-#  windows_container:
-#    image: cirrusci/windowsservercore:2019
-#  env:
-#    CIRRUS_SHELL: powershell
-#    PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%'
-#    PYTHONUTF8: 1
-#    QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
-#    QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
-#    QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
-#    VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
-#    VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74'
-#  install_script:
-#    - choco install python --version=3.7.7 -y
 
 task:
-  name: 'lint [bionic]'
-  << : *BASE_TEMPLATE
+  name: "x86_64: Linux (Debian stable)"
   container:
-    image: ubuntu:bionic  # For python 3.6, oldest supported version according to doc/dependencies.md
+    dockerfile: ci/linux-debian.Dockerfile
+    # Reduce number of CPUs to be able to do more builds in parallel.
     cpu: 1
+    # More than enough for our scripts.
     memory: 1G
-  # For faster CI feedback, immediately schedule the linters
-  << : *CREDITS_TEMPLATE
-  lint_script:
-    - ./ci/lint_run_all.sh
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-
-task:
-  name: 'ARM [unit tests, no functional tests] [buster]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: debian:buster
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_arm.sh"
-
-task:
-  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_win64.sh"
-
-task:
-  name: '32-bit + dash [gui] [CentOS 8]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: centos:8
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    PACKAGE_MANAGER_INSTALL: "yum install -y"
-    FILE_ENV: "./ci/test/00_setup_env_i686_centos.sh"
-
-task:
-  name: '[previous releases, uses qt5 dev package and some depends packages, DEBUG] [unsigned char] [bionic]'
-  previous_releases_cache:
-    folder: "releases"
-  << : *GLOBAL_TASK_TEMPLATE
-  << : *PERSISTENT_WORKER_TEMPLATE
-  env:
-    << : *PERSISTENT_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
-
-task:
-  name: '[depends, sanitizers: thread (TSan), no gui] [hirsute]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:hirsute
-    cpu: 6  # Increase CPU and Memory to avoid timeout
-    memory: 24G
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j8"
-    FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
-
-task:
-  name: '[depends, sanitizers: memory (MSan)] [focal]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+  matrix: &ENV_MATRIX
+    - env: {WIDEMUL:  int64,  RECOVERY: yes}
+    - env: {WIDEMUL:  int64,                 ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {WIDEMUL: int128}
+    - env: {WIDEMUL: int128,  RECOVERY: yes,            EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {WIDEMUL: int128,                 ECDH: yes, EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {WIDEMUL: int128,  ASM: x86_64}
+    - env: {                  RECOVERY: yes,            EXPERIMENTAL: yes, SCHNORRSIG: yes}
+    - env: {                  STATICPRECOMPUTATION: no}
+    - env: {BUILD: distcheck, WITH_VALGRIND: no, CTIMETEST: no, BENCH: no}
+    - env: {CPPFLAGS: -DDETERMINISTIC}
+    - env: {CFLAGS: -O0, CTIMETEST: no}
+    - env:
+        CFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+        LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+        UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+        ASM: x86_64
+        ECDH: yes
+        RECOVERY: yes
+        EXPERIMENTAL: yes
+        SCHNORRSIG: yes
+        CTIMETEST: no
+    - env: { ECMULTGENPRECISION: 2 }
+    - env: { ECMULTGENPRECISION: 8 }
+    - env:
+        RUN_VALGRIND: yes
+        ASM: x86_64
+        ECDH: yes
+        RECOVERY: yes
+        EXPERIMENTAL: yes
+        SCHNORRSIG: yes
+        EXTRAFLAGS: "--disable-openssl-tests"
+        BUILD:
+  matrix:
+    - env:
+        CC: gcc
+    - env:
+        CC: clang
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
-  name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]'
-  << : *GLOBAL_TASK_TEMPLATE
+  name: "i686: Linux (Debian stable)"
   container:
-    image: ubuntu:hirsute
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
-
-task:
-  name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
-    cpu: 4  # Increase CPU and memory to avoid timeout
-    memory: 16G
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j8"
-    FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
-
-task:
-  name: '[multiprocess, DEBUG] [focal]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
-    cpu: 4
-    memory: 16G  # The default memory is sometimes just a bit too small, so double everything
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j8"
-    FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
-
-task:
-  name: '[no wallet] [bionic]'
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:bionic
-  env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh"
-
-task:
-  name: 'macOS 10.14 [gui, no tests] [focal]'
-  << : *DEPENDS_SDK_CACHE_TEMPLATE
-  << : *GLOBAL_TASK_TEMPLATE
-  container:
-    image: ubuntu:focal
+    dockerfile: ci/linux-debian.Dockerfile
+    cpu: 1
+    memory: 1G
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_mac.sh"
+    HOST: i686-linux-gnu
+    ECDH: yes
+    RECOVERY: yes
+    EXPERIMENTAL: yes
+    SCHNORRSIG: yes
+  matrix:
+    - env:
+        CC: i686-linux-gnu-gcc
+    - env:
+        CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
-  name: 'macOS 11 native [gui] [no depends]'
-  brew_install_script:
-    - brew install boost libevent berkeley-db4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
-  << : *GLOBAL_TASK_TEMPLATE
-  osx_instance:
-    # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
-    image: big-sur-xcode-12.5  # https://cirrus-ci.org/guide/macOS
+  name: "x86_64: macOS Catalina"
+  macos_instance:
+    image: catalina-base
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    CI_USE_APT_INSTALL: "no"
-    PACKAGE_MANAGER_INSTALL: "echo"  # Nothing to do
-    FILE_ENV: "./ci/test/00_setup_env_mac_host.sh"
+    HOMEBREW_NO_AUTO_UPDATE: 1
+    HOMEBREW_NO_INSTALL_CLEANUP: 1
+    # Cirrus gives us a fixed number of 12 virtual CPUs. Not that we even have that many jobs at the moment...
+    MAKEFLAGS: -j13
+  matrix:
+    << : *ENV_MATRIX
+  matrix:
+    - env:
+        CC: gcc-9
+    - env:
+        CC: clang
+  # Update Command Line Tools
+  # Uncomment this if the Command Line Tools on the CirrusCI macOS image are too old to brew valgrind.
+  # See https://apple.stackexchange.com/a/195963 for the implementation.
+  ## update_clt_script:
+  ##   - system_profiler SPSoftwareDataType
+  ##   - touch /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress
+  ##   - |-
+  ##     PROD=$(softwareupdate -l | grep "*.*Command Line" | tail -n 1 | awk -F"*" '{print $2}' | sed -e 's/^ *//' | sed 's/Label: //g' | tr -d '\n')
+  ##   # For debugging
+  ##   - softwareupdate -l && echo "PROD: $PROD"
+  ##   - softwareupdate -i "$PROD" --verbose
+  ##   - rm /tmp/.com.apple.dt.CommandLineTools.installondemand.in-progress
+  ##
+  brew_valgrind_pre_script:
+    - brew config
+    - brew tap --shallow LouisBrunner/valgrind
+    # Fetch valgrind source but don't build it yet.
+    - brew fetch --HEAD LouisBrunner/valgrind/valgrind
+  brew_valgrind_cache:
+    # This is $(brew --cellar valgrind) but command substition does not work here.
+    folder: /usr/local/Cellar/valgrind
+    # Rebuild cache if ...
+    fingerprint_script:
+      # ... macOS version changes:
+      - sw_vers
+      # ... brew changes:
+      - brew config
+      # ... valgrind changes:
+      - git -C "$(brew --cache)/valgrind--git" rev-parse HEAD
+    populate_script:
+      # If there's no hit in the cache, build and install valgrind.
+      - brew install --HEAD LouisBrunner/valgrind/valgrind
+  brew_valgrind_post_script:
+    # If we have restored valgrind from the cache, tell brew to create symlink to the PATH.
+    # If we haven't restored from cached (and just run brew install), this is a no-op.
+    - brew link valgrind
+  brew_script:
+    - brew install automake libtool gcc@9
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
 
 task:
-  name: 'ARM64 Android APK [focal]'
-  << : *DEPENDS_SDK_CACHE_TEMPLATE
-  depends_sources_cache:
-    folder: "depends/sources"
-  << : *GLOBAL_TASK_TEMPLATE
+  name: "s390x (big-endian): Linux (Debian stable, QEMU)"
   container:
-    image: ubuntu:focal
+    dockerfile: ci/linux-debian.Dockerfile
+    cpu: 1
+    memory: 1G
   env:
-    << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_android.sh"
+    QEMU_CMD: qemu-s390x
+    HOST: s390x-linux-gnu
+    BUILD:
+    WITH_VALGRIND: no
+    ECDH: yes
+    RECOVERY: yes
+    EXPERIMENTAL: yes
+    SCHNORRSIG: yes
+    CTIMETEST: no
+  << : *MERGE_BASE
+  test_script:
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=27008
+    - rm /etc/ld.so.cache
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS