dashpay · PastaPastaPasta · Oct 24, 2024 · Oct 20, 2024 · Oct 20, 2024 · Jul 20, 2021
diff --git a/Makefile.am b/Makefile.am
@@ -177,6 +177,7 @@ LCOV_FILTER_PATTERN = \
 	-p "src/bench/" \
 	-p "src/univalue" \
 	-p "src/crypto/ctaes" \
+	-p "src/minisketch" \
 	-p "src/secp256k1" \
 	-p "depends"
 

diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh
@@ -17,6 +17,7 @@ export COMMIT_RANGE
 # check with -r to not have to fetch all the remotes.
 test/lint/git-subtree-check.sh src/crypto/ctaes
 test/lint/git-subtree-check.sh src/secp256k1
+test/lint/git-subtree-check.sh src/minisketch
 test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/check-doc.py

diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do
     # shellcheck disable=SC2044
     for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
       if (file "$b" | grep "Windows"); then

diff --git a/configure.ac b/configure.ac
@@ -537,6 +537,24 @@ AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERRO
 AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]])
 AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[X86_SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]])
 
+enable_clmul=
+AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([
+  #include <stdint.h>
+  #include <x86intrin.h>
+], [
+  __m128i a = _mm_cvtsi64_si128((uint64_t)7);
+  __m128i b = _mm_clmulepi64_si128(a, a, 37);
+  __m128i c = _mm_srli_epi64(b, 41);
+  __m128i d = _mm_xor_si128(b, c);
+  uint64_t e = _mm_cvtsi128_si64(d);
+  return e == 0;
+])])
+
+if test x$enable_clmul = xyes; then
+  CLMUL_CXXFLAGS="-mpclmul"
+  AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used])
+fi
+
 TEMP_CXXFLAGS="$CXXFLAGS"
 CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
 AC_MSG_CHECKING(for SSE4.2 intrinsics)
@@ -1030,19 +1048,20 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
                  #endif])
 
 AC_MSG_CHECKING(for __builtin_clzl)
+
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
  (void) __builtin_clzl(0);
   ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
- [ AC_MSG_RESULT(no)]
+ [ AC_MSG_RESULT(yes); have_clzl=yes; AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
+ [ AC_MSG_RESULT(no); have_clzl=no;]
 )
 
 AC_MSG_CHECKING(for __builtin_clzll)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
   (void) __builtin_clzll(0);
   ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
- [ AC_MSG_RESULT(no)]
+ [ AC_MSG_RESULT(yes); have_clzll=yes; AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
+ [ AC_MSG_RESULT(no); have_clzll=no;]
 )
 
 dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas)
@@ -1815,6 +1834,10 @@ AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes])
 AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes])
 AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes])
 
+dnl for minisketch
+AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes])
+AM_CONDITIONAL([HAVE_CLZ],[test x$have_clzl$have_clzll = xyesyes])
+
 AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
 AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])
 AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build])
@@ -1857,6 +1880,7 @@ AC_SUBST(SANITIZER_CXXFLAGS)
 AC_SUBST(SANITIZER_LDFLAGS)
 AC_SUBST(SSE42_CXXFLAGS)
 AC_SUBST(SSE41_CXXFLAGS)
+AC_SUBST(CLMUL_CXXFLAGS)
 AC_SUBST(AVX2_CXXFLAGS)
 AC_SUBST(X86_SHANI_CXXFLAGS)
 AC_SUBST(ARM_CRC_CXXFLAGS)

diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py
@@ -44,6 +44,7 @@
     "src/gsl/",
     "src/immer/",
     "src/leveldb/",
+    "src/minisketch",
     "src/secp256k1/",
     "src/univalue/",
 ]

diff --git a/doc/developer-notes.md b/doc/developer-notes.md
@@ -1063,6 +1063,9 @@ Current subtrees include:
 - src/univalue
   - Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue
 
+- src/minisketch
+  - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors.
+
 Upgrading LevelDB
 ---------------------
 

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -36,7 +36,7 @@ BACKTRACE_LIB = -lbacktrace
 endif
 endif #ENABLE_STACKTRACES
 
-BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
+BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
 BITCOIN_INCLUDES+=-isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include
 BITCOIN_INCLUDES+=-isystem$(srcdir)/immer
 
@@ -259,6 +259,7 @@ BITCOIN_CORE_H = \
   merkleblock.h \
   messagesigner.h \
   miner.h \
+  minisketchwrapper.h \
   net.h \
   net_permissions.h \
   net_processing.h \
@@ -494,6 +495,7 @@ libbitcoin_server_a_SOURCES = \
   masternode/sync.cpp \
   masternode/utils.cpp \
   miner.cpp \
+  minisketchwrapper.cpp \
   net.cpp \
   netfulfilledman.cpp \
   netgroup.cpp \
@@ -1059,8 +1061,11 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
 CLEANFILES += $(libbitcoin_ipc_mpgen_output)
 endif
 
+include Makefile.minisketch.include
+
 include Makefile.crc32c.include
 include Makefile.leveldb.include
+
 include Makefile.test_util.include
 include Makefile.test_fuzz.include
 

diff --git a/src/Makefile.minisketch.include b/src/Makefile.minisketch.include
@@ -0,0 +1,43 @@
+include minisketch/sources.mk
+
+LIBMINISKETCH_CPPFLAGS=
+LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32
+
+LIBMINISKETCH = minisketch/libminisketch.a
+MINISKETCH_LIBS = $(LIBMINISKETCH)
+
+if ENABLE_CLMUL
+LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a
+LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL
+MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL)
+endif
+
+if HAVE_CLZ
+LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ
+endif
+
+EXTRA_LIBRARIES += $(MINISKETCH_LIBS)
+
+minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT)
+minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS)
+minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+
+minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT)
+minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT)
+minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+
+if ENABLE_TESTS
+if !ENABLE_FUZZ
+MINISKETCH_TEST = minisketch/test
+TESTS += $(MINISKETCH_TEST)
+noinst_PROGRAMS += $(MINISKETCH_TEST)
+
+minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
+minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+minisketch_test_LDADD = $(MINISKETCH_LIBS)
+minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
+
+endif
+endif
diff --git a/src/Makefile.test.include b/src/Makefile.test.include
@@ -130,6 +130,7 @@ BITCOIN_TESTS =\
   test/mempool_tests.cpp \
   test/merkle_tests.cpp \
   test/merkleblock_tests.cpp \
+  test/minisketch_tests.cpp \
   test/miner_tests.cpp \
   test/multisig_tests.cpp \
   test/net_peer_eviction_tests.cpp \
@@ -219,7 +220,7 @@ if ENABLE_WALLET
 test_test_dash_LDADD += $(LIBBITCOIN_WALLET)
 endif
 test_test_dash_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
+  $(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS)
 test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
 
 test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS)

diff --git a/src/minisketch/.cirrus.yml b/src/minisketch/.cirrus.yml
@@ -0,0 +1,154 @@
+env:
+  BUILD: check
+  HOST:
+  MAKEFLAGS: -j4
+  BENCH: yes
+  TESTRUNS:
+  EXEC_CMD:
+  ENABLE_FIELDS:
+
+cat_logs_snippet: &CAT_LOGS
+  on_failure:
+    cat_test_log_script:
+      - cat test-suite.log || true
+    cat_config_log_script:
+      - cat config.log || true
+    cat_test_env_script:
+      - cat test_env.log || true
+    cat_ci_env_script:
+      - env
+
+merge_base_script_snippet: &MERGE_BASE
+  merge_base_script:
+    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
+    - git config --global user.email "ci@ci.ci"
+    - git config --global user.name "ci"
+    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
+
+env_matrix_snippet: &ENV_MATRIX_VALGRIND
+  - env:
+      ENABLE_FIELDS: "7,32,58"
+  - env:
+      BUILD: distcheck
+  - env:
+      EXEC_CMD: valgrind --error-exitcode=42
+      TESTRUNS: 1
+      BUILD:
+
+env_matrix_snippet: &ENV_MATRIX_SAN
+  - env:
+      ENABLE_FIELDS: 28
+  - env:
+      BUILD: distcheck
+  - env:
+      CXXFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+      LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+      BENCH: no
+
+env_matrix_snippet: &ENV_MATRIX_SAN_VALGRIND
+  - env:
+      ENABLE_FIELDS: "11,64,37"
+  - env:
+      BUILD: distcheck
+  - env:
+      EXEC_CMD: valgrind --error-exitcode=42
+      TESTRUNS: 1
+      BUILD:
+  - env:
+      CXXFLAGS:  "-fsanitize=undefined -fno-omit-frame-pointer"
+      LDFLAGS: "-fsanitize=undefined -fno-omit-frame-pointer"
+      UBSAN_OPTIONS: "print_stacktrace=1:halt_on_error=1"
+      BENCH: no
+
+task:
+  name: "x86_64: Linux (Debian stable)"
+  container:
+    dockerfile: ci/linux-debian.Dockerfile
+    memory: 2G
+    cpu: 4
+  matrix:
+    << : *ENV_MATRIX_SAN_VALGRIND
+  matrix:
+    - env:
+        CC: gcc
+    - env:
+        CC: clang
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "i686: Linux (Debian stable)"
+  container:
+    dockerfile: ci/linux-debian.Dockerfile
+    memory: 2G
+    cpu: 4
+  env:
+    HOST: i686-linux-gnu
+  matrix:
+    << : *ENV_MATRIX_VALGRIND
+  matrix:
+    - env:
+        CC: i686-linux-gnu-gcc
+    - env:
+        CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "x86_64: macOS Catalina"
+  macos_instance:
+    image: catalina-base
+  env:
+    # Cirrus gives us a fixed number of 12 virtual CPUs.
+    MAKEFLAGS: -j13
+  matrix:
+    << : *ENV_MATRIX_SAN
+  matrix:
+    - env:
+        CC: gcc-9
+    - env:
+        CC: clang
+  brew_script:
+    - brew update
+    - brew install automake libtool gcc@9
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "s390x (big-endian): Linux (Debian stable, QEMU)"
+  container:
+    dockerfile: ci/linux-debian.Dockerfile
+    cpu: 4
+    memory: 2G
+  env:
+    EXEC_CMD: qemu-s390x -L /usr/s390x-linux-gnu
+    HOST: s390x-linux-gnu
+    BUILD:
+  << : *MERGE_BASE
+  test_script:
+    # https://sourceware.org/bugzilla/show_bug.cgi?id=27008
+    - rm /etc/ld.so.cache
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS
+
+task:
+  name: "x86_64-w64-mingw32: Linux (Debian stable, Wine)"
+  container:
+    dockerfile: ci/linux-debian.Dockerfile
+    cpu: 4
+    memory: 2G
+  env:
+    EXEC_CMD: wine
+    HOST: x86_64-w64-mingw32
+    BUILD:
+  << : *MERGE_BASE
+  test_script:
+    - ./ci/cirrus.sh
+  << : *CAT_LOGS