backport: merge bitcoin#23842, #24714, #21059, #23994, #19866, #19935, #23756, #19064, #23411, #20080 (util backports)

configure.ac

@@ -132,6 +132,12 @@ AC_ARG_WITH([bdb],
   [use_bdb=$withval],
   [use_bdb=auto])
 
+AC_ARG_ENABLE([ebpf],
+  [AS_HELP_STRING([--enable-ebpf],
+  [enable eBPF tracing (default is yes if sys/sdt.h is found)])],
+  [use_ebpf=$enableval],
+  [use_ebpf=yes])
+
 AC_ARG_WITH([miniupnpc],
   [AS_HELP_STRING([--with-miniupnpc],
   [enable UPNP (default is yes if libminiupnpc is found)])],
@@ -1368,6 +1374,16 @@ if test x$enable_wallet != xno; then
     fi
 fi
 
+if test x$use_ebpf != xno; then
+  AC_CHECK_HEADER([sys/sdt.h], [have_sdt=yes], [have_sdt=no])
+else
+  have_sdt=no
+fi
+
+if test x$have_sdt = xyes; then
+  AC_DEFINE([ENABLE_TRACING], [1], [Define to 1 to enable eBPF user static defined tracepoints])
+fi
+
 dnl Check for libminiupnpc (optional)
 if test x$use_upnp != xno; then
   AC_CHECK_HEADERS(
@@ -1740,6 +1756,7 @@ AM_CONDITIONAL([TARGET_WINDOWS], [test x$TARGET_OS = xwindows])
 AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes])
 AM_CONDITIONAL([USE_SQLITE], [test "x$use_sqlite" = "xyes"])
 AM_CONDITIONAL([USE_BDB], [test "x$use_bdb" = "xyes"])
+AM_CONDITIONAL([ENABLE_TRACING],[test x$have_sdt = xyes])
 AM_CONDITIONAL([ENABLE_TESTS],[test x$BUILD_TEST = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ],[test x$enable_fuzz = xyes])
 AM_CONDITIONAL([ENABLE_FUZZ_BINARY],[test x$enable_fuzz_binary = xyes])
@@ -1905,6 +1922,7 @@ echo "  with bench    = $use_bench"
 echo "  with upnp           = $use_upnp"
 echo "  with natpmp         = $use_natpmp"
 echo "  use asm             = $use_asm"
+echo "  ebpf tracing        = $have_sdt"
 echo "  sanitizers          = $use_sanitizers"
 echo "  debug enabled       = $enable_debug"
 echo "  stacktraces enabled = $enable_stacktraces"

src/Makefile.am

@@ -310,8 +310,10 @@ BITCOIN_CORE_H = \
   util/check.h \
   util/enumerate.h \
   util/error.h \
+  util/fastrange.h \
   util/fees.h \
   util/golombrice.h \
+  util/hasher.h \
   util/hash_type.h \
   util/irange.h \
   util/spanparsing.h \
@@ -321,6 +323,7 @@ BITCOIN_CORE_H = \
   util/macros.h \
   util/message.h \
   util/moneystr.h \
+  util/overflow.h \
   util/ranges.h \
   util/readwritefile.h \
   util/underlying.h \
@@ -330,7 +333,9 @@ BITCOIN_CORE_H = \
   util/sock.h \
   util/string.h \
   util/time.h \
+  util/thread.h \
   util/threadnames.h \
+  util/trace.h \
   util/translation.h \
   util/vector.h \
   util/url.h \
@@ -735,8 +740,10 @@ libbitcoin_util_a_SOURCES = \
   util/asmap.cpp \
   util/bip32.cpp \
   util/bytevectorhash.cpp \
+  util/check.cpp \
   util/error.cpp \
   util/fees.cpp \
+  util/hasher.cpp \
   util/getuniquepath.cpp \
   util/sock.cpp \
   util/system.cpp \
@@ -750,6 +757,7 @@ libbitcoin_util_a_SOURCES = \
   util/time.cpp \
   util/serfloat.cpp \
   util/string.cpp \
+  util/thread.cpp \
   util/threadnames.cpp \
   $(BITCOIN_CORE_H)
 

src/bench/bench.h

@@ -5,15 +5,15 @@
 #ifndef BITCOIN_BENCH_BENCH_H
 #define BITCOIN_BENCH_BENCH_H
 
+#include <util/macros.h>
+
 #include <chrono>
 #include <functional>
 #include <map>
 #include <string>
 #include <vector>
 
 #include <bench/nanobench.h>
-#include <boost/preprocessor/cat.hpp>
-#include <boost/preprocessor/stringize.hpp>
 
 /*
  * Usage:
@@ -56,8 +56,8 @@ class BenchRunner
     static void RunAll(const Args& args);
 };
 }
-// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo");
+// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo", foo);
 #define BENCHMARK(n) \
-    benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n);
+    benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n);
 
 #endif // BITCOIN_BENCH_BENCH_H

src/blockfilter.cpp

@@ -29,7 +29,7 @@ uint64_t GCSFilter::HashToRange(const Element& element) const
     uint64_t hash = CSipHasher(m_params.m_siphash_k0, m_params.m_siphash_k1)
         .Write(element.data(), element.size())
         .Finalize();
-    return MapIntoRange(hash, m_F);
+    return FastRange64(hash, m_F);
 }
 
 std::vector<uint64_t> GCSFilter::BuildHashedSet(const ElementSet& elements) const

src/bloom.cpp

@@ -14,6 +14,7 @@
 #include <script/standard.h>
 #include <random.h>
 #include <streams.h>
+#include <util/fastrange.h>
 
 #include <math.h>
 #include <stdlib.h>
@@ -307,14 +308,6 @@ static inline uint32_t RollingBloomHash(unsigned int nHashNum, uint32_t nTweak,
     return MurmurHash3(nHashNum * 0xFBA4C795 + nTweak, vDataToHash);
 }
 
-
-// A replacement for x % n. This assumes that x and n are 32bit integers, and x is a uniformly random distributed 32bit value
-// which should be the case for a good hash.
-// See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
-static inline uint32_t FastMod(uint32_t x, size_t n) {
-    return ((uint64_t)x * (uint64_t)n) >> 32;
-}
-
 void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey)
 {
     if (nEntriesThisGeneration == nEntriesPerGeneration) {
@@ -339,7 +332,7 @@ void CRollingBloomFilter::insert(const std::vector<unsigned char>& vKey)
         uint32_t h = RollingBloomHash(n, nTweak, vKey);
         int bit = h & 0x3F;
         /* FastMod works with the upper bits of h, so it is safe to ignore that the lower bits of h are already used for bit. */
-        uint32_t pos = FastMod(h, data.size());
+        uint32_t pos = FastRange32(h, data.size());
         /* The lowest bit of pos is ignored, and set to zero for the first bit, and to one for the second. */
         data[pos & ~1] = (data[pos & ~1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration & 1)) << bit;
         data[pos | 1] = (data[pos | 1] & ~(((uint64_t)1) << bit)) | ((uint64_t)(nGeneration >> 1)) << bit;
@@ -357,7 +350,7 @@ bool CRollingBloomFilter::contains(const std::vector<unsigned char>& vKey) const
     for (int n = 0; n < nHashFuncs; n++) {
         uint32_t h = RollingBloomHash(n, nTweak, vKey);
         int bit = h & 0x3F;
-        uint32_t pos = FastMod(h, data.size());
+        uint32_t pos = FastRange32(h, data.size());
         /* If the relevant bit is not set in either data[pos & ~1] or data[pos | 1], the filter does not contain vKey */
         if (!(((data[pos & ~1] | data[pos | 1]) >> bit) & 1)) {
             return false;

src/clientversion.h

@@ -5,6 +5,8 @@
 #ifndef BITCOIN_CLIENTVERSION_H
 #define BITCOIN_CLIENTVERSION_H
 
+#include <util/macros.h>
+
 #if defined(HAVE_CONFIG_H)
 #include <config/bitcoin-config.h>
 #endif //HAVE_CONFIG_H
@@ -14,13 +16,6 @@
 #error Client version information missing: version is not defined by bitcoin-config.h or in any other way
 #endif
 
-/**
- * Converts the parameter X to a string after macro replacement on X has been performed.
- * Don't merge these into one macro!
- */
-#define STRINGIZE(X) DO_STRINGIZE(X)
-#define DO_STRINGIZE(X) #X
-
 //! Copyright string used in Windows .rc files
 #define COPYRIGHT_STR "2009-" STRINGIZE(COPYRIGHT_YEAR) " The Bitcoin Core Developers, 2014-" STRINGIZE(COPYRIGHT_YEAR) " " COPYRIGHT_HOLDERS_FINAL
 

src/coins.cpp

@@ -31,8 +31,6 @@ bool CCoinsViewBacked::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock)
 CCoinsViewCursor *CCoinsViewBacked::Cursor() const { return base->Cursor(); }
 size_t CCoinsViewBacked::EstimateSize() const { return base->EstimateSize(); }
 
-SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand(std::numeric_limits<uint64_t>::max())), k1(GetRand(std::numeric_limits<uint64_t>::max())) {}
-
 CCoinsViewCache::CCoinsViewCache(CCoinsView *baseIn) : CCoinsViewBacked(baseIn), cachedCoinsUsage(0) {}
 
 size_t CCoinsViewCache::DynamicMemoryUsage() const {

src/coins.h

@@ -8,11 +8,11 @@
 
 #include <compressor.h>
 #include <core_memusage.h>
-#include <crypto/siphash.h>
 #include <memusage.h>
 #include <primitives/transaction.h>
 #include <serialize.h>
 #include <uint256.h>
+#include <util/hasher.h>
 
 #include <assert.h>
 #include <stdint.h>
@@ -85,33 +85,6 @@ class Coin
     }
 };
 
-class SaltedOutpointHasher
-{
-private:
-    /** Salt */
-    const uint64_t k0, k1;
-
-public:
-    SaltedOutpointHasher();
-
-    /**
-     * This *must* return size_t. With Boost 1.46 on 32-bit systems the
-     * unordered_map will behave unpredictably if the custom hasher returns a
-     * uint64_t, resulting in failures when syncing the chain (#4634).
-     *
-     * Having the hash noexcept allows libstdc++'s unordered_map to recalculate
-     * the hash during rehash, so it does not have to cache the value. This
-     * reduces node's memory by sizeof(size_t). The required recalculation has
-     * a slight performance penalty (around 1.6%), but this is compensated by
-     * memory savings of about 9% which allow for a larger dbcache setting.
-     *
-     * @see https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html
-     */
-    size_t operator()(const COutPoint& id) const noexcept {
-        return SipHashUint256Extra(k0, k1, id.hash, id.n);
-    }
-};
-
 /**
  * A Coin in one level of the coins database caching hierarchy.
  *

src/cuckoocache.h

@@ -5,6 +5,8 @@
 #ifndef BITCOIN_CUCKOOCACHE_H
 #define BITCOIN_CUCKOOCACHE_H
 
+#include <util/fastrange.h>
+
 #include <array>
 #include <algorithm> // std::find
 #include <atomic>
@@ -219,13 +221,8 @@ class cache
      * One option would be to implement the same trick the compiler uses and compute the
      *  constants for exact division based on the size, as described in "{N}-bit Unsigned
      *  Division via {N}-bit Multiply-Add" by Arch D. Robison in 2005. But that code is
-     *  somewhat complicated and the result is still slower than other options:
-     *
-     * Instead we treat the 32-bit random number as a Q32 fixed-point number in the range
-     *  [0, 1) and simply multiply it by the size. Then we just shift the result down by
-     *  32-bits to get our bucket number. The result has non-uniformity the same as a
-     *  mod, but it is much faster to compute. More about this technique can be found at
-     *  http://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ .
+     *  somewhat complicated and the result is still slower than an even simpler option:
+     *  see the FastRange32 function in util/fastrange.h.
      *
      * The resulting non-uniformity is also more equally distributed which would be
      *  advantageous for something like linear probing, though it shouldn't matter
@@ -241,14 +238,14 @@ class cache
      */
     inline std::array<uint32_t, 8> compute_hashes(const Element& e) const
     {
-        return {{(uint32_t)(((uint64_t)hash_function.template operator()<0>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<1>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<2>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<3>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<4>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<5>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<6>(e) * (uint64_t)size) >> 32),
-                 (uint32_t)(((uint64_t)hash_function.template operator()<7>(e) * (uint64_t)size) >> 32)}};
+        return {{FastRange32(hash_function.template operator()<0>(e), size),
+                 FastRange32(hash_function.template operator()<1>(e), size),
+                 FastRange32(hash_function.template operator()<2>(e), size),
+                 FastRange32(hash_function.template operator()<3>(e), size),
+                 FastRange32(hash_function.template operator()<4>(e), size),
+                 FastRange32(hash_function.template operator()<5>(e), size),
+                 FastRange32(hash_function.template operator()<6>(e), size),
+                 FastRange32(hash_function.template operator()<7>(e), size)}};
     }
 
     /** invalid returns a special index that can never be inserted to

src/index/base.cpp

@@ -8,6 +8,7 @@
 #include <shutdown.h>
 #include <tinyformat.h>
 #include <ui_interface.h>
+#include <util/thread.h>
 #include <util/translation.h>
 #include <validation.h> // For g_chainman
 #include <warnings.h>
@@ -353,8 +354,7 @@ bool BaseIndex::Start(CChainState& active_chainstate)
         return false;
     }
 
-    m_thread_sync = std::thread(&TraceThread<std::function<void()>>, GetName(),
-                                std::bind(&BaseIndex::ThreadSync, this));
+    m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); });
     return true;
 }
 

src/index/blockfilterindex.h

@@ -9,15 +9,11 @@
 #include <chain.h>
 #include <flatfile.h>
 #include <index/base.h>
+#include <util/hasher.h>
 
 /** Interval between compact filter checkpoints. See BIP 157. */
 static constexpr int CFCHECKPT_INTERVAL = 1000;
 
-struct FilterHeaderHasher
-{
-    size_t operator()(const uint256& hash) const { return ReadLE64(hash.begin()); }
-};
-
 /**
  * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of
  * blocks by height. An index is constructed for each supported filter type with its own database

src/index/coinstatsindex.cpp

@@ -321,7 +321,7 @@ bool CoinStatsIndex::LookUpStats(const CBlockIndex* block_index, CCoinsStats& co
     coins_stats.hashSerialized = entry.muhash;
     coins_stats.nTransactionOutputs = entry.transaction_output_count;
     coins_stats.nBogoSize = entry.bogo_size;
-    coins_stats.nTotalAmount = entry.total_amount;
+    coins_stats.total_amount = entry.total_amount;
     coins_stats.total_subsidy = entry.total_subsidy;
     coins_stats.total_unspendable_amount = entry.total_unspendable_amount;
     coins_stats.total_prevout_spent_amount = entry.total_prevout_spent_amount;