Skip to content

Commit a6b26b5

Browse files
kwvgkwvg
committed
merge bitcoin#23114: Add minisketch subtree and integrate into build/test
excludes: - b6487dc (subtree manipulation done in previous commits) - 07f0a61 (see above) - 0eb7928 (we don't support MSVC build method)
1 parent 57ef0db commit a6b26b5

23 files changed

+243
-17
lines changed

‎Makefile.am‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,7 @@ LCOV_FILTER_PATTERN = \
177177
-p "src/bench/" \
178178
-p "src/univalue" \
179179
-p "src/crypto/ctaes" \
180+
-p "src/minisketch" \
180181
-p "src/secp256k1" \
181182
-p "depends"
182183

‎ci/lint/06_script.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ export COMMIT_RANGE
1717
# check with -r to not have to fetch all the remotes.
1818
test/lint/git-subtree-check.sh src/crypto/ctaes
1919
test/lint/git-subtree-check.sh src/secp256k1
20+
test/lint/git-subtree-check.sh src/minisketch
2021
test/lint/git-subtree-check.sh src/univalue
2122
test/lint/git-subtree-check.sh src/leveldb
2223
test/lint/check-doc.py

‎ci/test/wrap-wine.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
export LC_ALL=C.UTF-8
88

9-
for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
9+
for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do
1010
# shellcheck disable=SC2044
1111
for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
1212
if (file "$b" | grep "Windows"); then

‎configure.ac‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,24 @@ AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERRO
537537
AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]])
538538
AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[X86_SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]])
539539

540+
enable_clmul=
541+
AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([
542+
#include <stdint.h>
543+
#include <x86intrin.h>
544+
], [
545+
__m128i a = _mm_cvtsi64_si128((uint64_t)7);
546+
__m128i b = _mm_clmulepi64_si128(a, a, 37);
547+
__m128i c = _mm_srli_epi64(b, 41);
548+
__m128i d = _mm_xor_si128(b, c);
549+
uint64_t e = _mm_cvtsi128_si64(d);
550+
return e == 0;
551+
])])
552+
553+
if test x$enable_clmul = xyes; then
554+
CLMUL_CXXFLAGS="-mpclmul"
555+
AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used])
556+
fi
557+
540558
TEMP_CXXFLAGS="$CXXFLAGS"
541559
CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
542560
AC_MSG_CHECKING(for SSE4.2 intrinsics)
@@ -1030,19 +1048,20 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
10301048
#endif])
10311049

10321050
AC_MSG_CHECKING(for __builtin_clzl)
1051+
10331052
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
10341053
(void) __builtin_clzl(0);
10351054
]])],
1036-
[ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
1037-
[ AC_MSG_RESULT(no)]
1055+
[ AC_MSG_RESULT(yes); have_clzl=yes; AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
1056+
[ AC_MSG_RESULT(no); have_clzl=no;]
10381057
)
10391058

10401059
AC_MSG_CHECKING(for __builtin_clzll)
10411060
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
10421061
(void) __builtin_clzll(0);
10431062
]])],
1044-
[ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
1045-
[ AC_MSG_RESULT(no)]
1063+
[ AC_MSG_RESULT(yes); have_clzll=yes; AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
1064+
[ AC_MSG_RESULT(no); have_clzll=no;]
10461065
)
10471066

10481067
dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas)
@@ -1815,6 +1834,10 @@ AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes])
18151834
AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes])
18161835
AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes])
18171836

1837+
dnl for minisketch
1838+
AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes])
1839+
AM_CONDITIONAL([HAVE_CLZ],[test x$have_clzl$have_clzll = xyesyes])
1840+
18181841
AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
18191842
AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])
18201843
AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build])
@@ -1857,6 +1880,7 @@ AC_SUBST(SANITIZER_CXXFLAGS)
18571880
AC_SUBST(SANITIZER_LDFLAGS)
18581881
AC_SUBST(SSE42_CXXFLAGS)
18591882
AC_SUBST(SSE41_CXXFLAGS)
1883+
AC_SUBST(CLMUL_CXXFLAGS)
18601884
AC_SUBST(AVX2_CXXFLAGS)
18611885
AC_SUBST(X86_SHANI_CXXFLAGS)
18621886
AC_SUBST(ARM_CRC_CXXFLAGS)

‎contrib/devtools/copyright_header.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
"src/gsl/",
4545
"src/immer/",
4646
"src/leveldb/",
47+
"src/minisketch",
4748
"src/secp256k1/",
4849
"src/univalue/",
4950
]

‎doc/developer-notes.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1063,6 +1063,9 @@ Current subtrees include:
10631063
- src/univalue
10641064
- Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue
10651065

1066+
- src/minisketch
1067+
- Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors.
1068+
10661069
Upgrading LevelDB
10671070
---------------------
10681071

‎src/Makefile.am‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ BACKTRACE_LIB = -lbacktrace
3636
endif
3737
endif #ENABLE_STACKTRACES
3838

39-
BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
39+
BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
4040
BITCOIN_INCLUDES+=-isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include
4141
BITCOIN_INCLUDES+=-isystem$(srcdir)/immer
4242

@@ -259,6 +259,7 @@ BITCOIN_CORE_H = \
259259
merkleblock.h \
260260
messagesigner.h \
261261
miner.h \
262+
minisketchwrapper.h \
262263
net.h \
263264
net_permissions.h \
264265
net_processing.h \
@@ -494,6 +495,7 @@ libbitcoin_server_a_SOURCES = \
494495
masternode/sync.cpp \
495496
masternode/utils.cpp \
496497
miner.cpp \
498+
minisketchwrapper.cpp \
497499
net.cpp \
498500
netfulfilledman.cpp \
499501
netgroup.cpp \
@@ -1059,8 +1061,11 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
10591061
CLEANFILES += $(libbitcoin_ipc_mpgen_output)
10601062
endif
10611063

1064+
include Makefile.minisketch.include
1065+
10621066
include Makefile.crc32c.include
10631067
include Makefile.leveldb.include
1068+
10641069
include Makefile.test_util.include
10651070
include Makefile.test_fuzz.include
10661071

‎src/Makefile.minisketch.include‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
include minisketch/sources.mk
2+
3+
LIBMINISKETCH_CPPFLAGS=
4+
LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32
5+
6+
LIBMINISKETCH = minisketch/libminisketch.a
7+
MINISKETCH_LIBS = $(LIBMINISKETCH)
8+
9+
if ENABLE_CLMUL
10+
LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a
11+
LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL
12+
MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL)
13+
endif
14+
15+
if HAVE_CLZ
16+
LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ
17+
endif
18+
19+
EXTRA_LIBRARIES += $(MINISKETCH_LIBS)
20+
21+
minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT)
22+
minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS)
23+
minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
24+
25+
minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT)
26+
minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT)
27+
minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
28+
minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
29+
30+
if ENABLE_TESTS
31+
if !ENABLE_FUZZ
32+
MINISKETCH_TEST = minisketch/test
33+
TESTS += $(MINISKETCH_TEST)
34+
noinst_PROGRAMS += $(MINISKETCH_TEST)
35+
36+
minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
37+
minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
38+
minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
39+
minisketch_test_LDADD = $(MINISKETCH_LIBS)
40+
minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
41+
42+
endif
43+
endif

‎src/Makefile.test.include‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,7 @@ BITCOIN_TESTS =\
130130
test/mempool_tests.cpp \
131131
test/merkle_tests.cpp \
132132
test/merkleblock_tests.cpp \
133+
test/minisketch_tests.cpp \
133134
test/miner_tests.cpp \
134135
test/multisig_tests.cpp \
135136
test/net_peer_eviction_tests.cpp \
@@ -219,7 +220,7 @@ if ENABLE_WALLET
219220
test_test_dash_LDADD += $(LIBBITCOIN_WALLET)
220221
endif
221222
test_test_dash_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
222-
$(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
223+
$(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS)
223224
test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
224225

225226
test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS)

‎src/minisketchwrapper.cpp‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
// Copyright (c) 2021 The Bitcoin Core developers
2+
// Distributed under the MIT software license, see the accompanying
3+
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
4+
5+
#include <minisketchwrapper.h>
6+
7+
#include <logging.h>
8+
#include <util/time.h>
9+
10+
#include <minisketch.h>
11+
12+
#include <algorithm>
13+
#include <cstddef>
14+
#include <cstdint>
15+
#include <optional>
16+
#include <utility>
17+
#include <vector>
18+
19+
namespace {
20+
21+
static constexpr uint32_t BITS = 32;
22+
23+
uint32_t FindBestImplementation()
24+
{
25+
std::optional<std::pair<int64_t, uint32_t>> best;
26+
27+
uint32_t max_impl = Minisketch::MaxImplementation();
28+
for (uint32_t impl = 0; impl <= max_impl; ++impl) {
29+
std::vector<int64_t> benches;
30+
uint64_t offset = 0;
31+
/* Run a little benchmark with capacity 32, adding 184 entries, and decoding 11 of them once. */
32+
for (int b = 0; b < 11; ++b) {
33+
if (!Minisketch::ImplementationSupported(BITS, impl)) break;
34+
Minisketch sketch(BITS, impl, 32);
35+
auto start = GetTimeMicros();
36+
for (uint64_t e = 0; e < 100; ++e) {
37+
sketch.Add(e*1337 + b*13337 + offset);
38+
}
39+
for (uint64_t e = 0; e < 84; ++e) {
40+
sketch.Add(e*1337 + b*13337 + offset);
41+
}
42+
offset += (*sketch.Decode(32))[0];
43+
auto stop = GetTimeMicros();
44+
benches.push_back(stop - start);
45+
}
46+
/* Remember which implementation has the best median benchmark time. */
47+
if (!benches.empty()) {
48+
std::sort(benches.begin(), benches.end());
49+
if (!best || best->first > benches[5]) {
50+
best = std::make_pair(benches[5], impl);
51+
}
52+
}
53+
}
54+
assert(best.has_value());
55+
LogPrintf("Using Minisketch implementation number %i\n", best->second);
56+
return best->second;
57+
}
58+
59+
uint32_t Minisketch32Implementation()
60+
{
61+
// Fast compute-once idiom.
62+
static uint32_t best = FindBestImplementation();
63+
return best;
64+
}
65+
66+
} // namespace
67+
68+
69+
Minisketch MakeMinisketch32(size_t capacity)
70+
{
71+
return Minisketch(BITS, Minisketch32Implementation(), capacity);
72+
}
73+
74+
Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits)
75+
{
76+
return Minisketch::CreateFP(BITS, Minisketch32Implementation(), max_elements, fpbits);
77+
}

0 commit comments

Comments
 (0)