bench: add "priority level" to the benchmark framework

src/Makefile.test.include

@@ -373,8 +373,8 @@ endif
 if TARGET_WINDOWS
 else
 if ENABLE_BENCH
-	@echo "Running bench/bench_bitcoin (one iteration sanity check)..."
-	$(BENCH_BINARY) --sanity-check > /dev/null
+	@echo "Running bench/bench_bitcoin (one iteration sanity check, only high priority)..."
+	$(BENCH_BINARY) -sanity-check -priority-level=high > /dev/null
 endif
 endif
 	$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check

src/bench/addrman.cpp

@@ -133,7 +133,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(AddrManAdd);
-BENCHMARK(AddrManSelect);
-BENCHMARK(AddrManGetAddr);
-BENCHMARK(AddrManAddThenGood);
+BENCHMARK(AddrManAdd, benchmark::PriorityLevel::HIGH);
+BENCHMARK(AddrManSelect, benchmark::PriorityLevel::HIGH);
+BENCHMARK(AddrManGetAddr, benchmark::PriorityLevel::HIGH);
+BENCHMARK(AddrManAddThenGood, benchmark::PriorityLevel::HIGH);

src/bench/base58.cpp

@@ -50,6 +50,6 @@ static void Base58Decode(benchmark::Bench& bench)
 }
 
 
-BENCHMARK(Base58Encode);
-BENCHMARK(Base58CheckEncode);
-BENCHMARK(Base58Decode);
+BENCHMARK(Base58Encode, benchmark::PriorityLevel::HIGH);
+BENCHMARK(Base58CheckEncode, benchmark::PriorityLevel::HIGH);
+BENCHMARK(Base58Decode, benchmark::PriorityLevel::HIGH);

src/bench/bech32.cpp

@@ -32,5 +32,5 @@ static void Bech32Decode(benchmark::Bench& bench)
 }
 
 
-BENCHMARK(Bech32Encode);
-BENCHMARK(Bech32Decode);
+BENCHMARK(Bech32Encode, benchmark::PriorityLevel::HIGH);
+BENCHMARK(Bech32Decode, benchmark::PriorityLevel::HIGH);

src/bench/bench.cpp

@@ -6,6 +6,7 @@
 
 #include <fs.h>
 #include <test/util/setup_common.h>
+#include <util/string.h>
 
 #include <chrono>
 #include <fstream>
@@ -41,18 +42,42 @@ void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& bench
 
 } // namespace
 
-benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks()
+namespace benchmark {
+
+// map a label to one or multiple priority levels
+std::map<std::string, uint8_t> map_label_priority = {
+    {"high", PriorityLevel::HIGH},
+    {"low", PriorityLevel::LOW},
+    {"all", 0xff}
+};
+
+std::string ListPriorities()
 {
-    static std::map<std::string, BenchFunction> benchmarks_map;
+    using item_t = std::pair<std::string, uint8_t>;
+    auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; };
+    std::set<item_t, decltype(sort_by_priority)> sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority);
+    return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; });
+}
+
+uint8_t StringToPriority(const std::string& str)
+{
+    auto it = map_label_priority.find(str);
+    if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str));
+    return it->second;
+}
+
+BenchRunner::BenchmarkMap& BenchRunner::benchmarks()
+{
+    static BenchmarkMap benchmarks_map;
     return benchmarks_map;
 }
 
-benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func)
+BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level)
 {
-    benchmarks().insert(std::make_pair(name, func));
+    benchmarks().insert(std::make_pair(name, std::make_pair(func, level)));
 }
 
-void benchmark::BenchRunner::RunAll(const Args& args)
+void BenchRunner::RunAll(const Args& args)
 {
     std::regex reFilter(args.regex_filter);
     std::smatch baseMatch;
@@ -62,33 +87,39 @@ void benchmark::BenchRunner::RunAll(const Args& args)
     }
 
     std::vector<ankerl::nanobench::Result> benchmarkResults;
-    for (const auto& p : benchmarks()) {
-        if (!std::regex_match(p.first, baseMatch, reFilter)) {
+    for (const auto& [name, bench_func] : benchmarks()) {
+        const auto& [func, priority_level] = bench_func;
+
+        if (!(priority_level & args.priority)) {
+            continue;
+        }
+
+        if (!std::regex_match(name, baseMatch, reFilter)) {
             continue;
         }
 
         if (args.is_list_only) {
-            std::cout << p.first << std::endl;
+            std::cout << name << std::endl;
             continue;
         }
 
         Bench bench;
         if (args.sanity_check) {
             bench.epochs(1).epochIterations(1);
         }
-        bench.name(p.first);
+        bench.name(name);
         if (args.min_time > 0ms) {
             // convert to nanos before dividing to reduce rounding errors
             std::chrono::nanoseconds min_time_ns = args.min_time;
             bench.minEpochTime(min_time_ns / bench.epochs());
         }
 
         if (args.asymptote.empty()) {
-            p.second(bench);
+            func(bench);
         } else {
             for (auto n : args.asymptote) {
                 bench.complexityN(n);
-                p.second(bench);
+                func(bench);
             }
             std::cout << bench.complexityBigO() << std::endl;
         }
@@ -103,3 +134,5 @@ void benchmark::BenchRunner::RunAll(const Args& args)
                                                                "{{/result}}");
     GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json());
 }
+
+} // namespace benchmark

src/bench/bench.h

@@ -41,6 +41,16 @@ using ankerl::nanobench::Bench;
 
 typedef std::function<void(Bench&)> BenchFunction;
 
+enum PriorityLevel : uint8_t
+{
+    LOW = 1 << 0,
+    HIGH = 1 << 2,
+};
+
+// List priority labels, comma-separated and sorted by increasing priority
+std::string ListPriorities();
+uint8_t StringToPriority(const std::string& str);
+
 struct Args {
     bool is_list_only;
     bool sanity_check;
@@ -49,22 +59,24 @@ struct Args {
     fs::path output_csv;
     fs::path output_json;
     std::string regex_filter;
+    uint8_t priority;
 };
 
 class BenchRunner
 {
-    typedef std::map<std::string, BenchFunction> BenchmarkMap;
+    // maps from "name" -> (function, priority_level)
+    typedef std::map<std::string, std::pair<BenchFunction, PriorityLevel>> BenchmarkMap;
     static BenchmarkMap& benchmarks();
 
 public:
-    BenchRunner(std::string name, BenchFunction func);
+    BenchRunner(std::string name, BenchFunction func, PriorityLevel level);
 
     static void RunAll(const Args& args);
 };
 } // namespace benchmark
 
-// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo", foo);
-#define BENCHMARK(n) \
-    benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n);
+// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo", foo, priority_level);
+#define BENCHMARK(n, priority_level) \
+    benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, priority_level);
 
 #endif // BITCOIN_BENCH_BENCH_H

src/bench/bench_bitcoin.cpp

@@ -18,6 +18,8 @@
 
 static const char* DEFAULT_BENCH_FILTER = ".*";
 static constexpr int64_t DEFAULT_MIN_TIME_MS{10};
+/** Priority level default value, run "all" priority levels */
+static const std::string DEFAULT_PRIORITY{"all"};
 
 static void SetupBenchArgs(ArgsManager& argsman)
 {
@@ -30,6 +32,8 @@ static void SetupBenchArgs(ArgsManager& argsman)
     argsman.AddArg("-output-csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-output-json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
     argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
+    argsman.AddArg("-priority-level=<l1,l2,l3>", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'",
+                                                           benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS);
 }
 
 // parses a comma separated list like "10,20,30,50"
@@ -45,6 +49,14 @@ static std::vector<double> parseAsymptote(const std::string& str) {
     return numbers;
 }
 
+static uint8_t parsePriorityLevel(const std::string& str) {
+    uint8_t levels{0};
+    for (const auto& level: SplitString(str, ',')) {
+        levels |= benchmark::StringToPriority(level);
+    }
+    return levels;
+}
+
 int main(int argc, char** argv)
 {
     ArgsManager argsman;
@@ -106,16 +118,22 @@ int main(int argc, char** argv)
         return EXIT_SUCCESS;
     }
 
-    benchmark::Args args;
-    args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", ""));
-    args.is_list_only = argsman.GetBoolArg("-list", false);
-    args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS));
-    args.output_csv = argsman.GetPathArg("-output-csv");
-    args.output_json = argsman.GetPathArg("-output-json");
-    args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
-    args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
+    try {
+        benchmark::Args args;
+        args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", ""));
+        args.is_list_only = argsman.GetBoolArg("-list", false);
+        args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS));
+        args.output_csv = argsman.GetPathArg("-output-csv");
+        args.output_json = argsman.GetPathArg("-output-json");
+        args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER);
+        args.sanity_check = argsman.GetBoolArg("-sanity-check", false);
+        args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY));
 
-    benchmark::BenchRunner::RunAll(args);
+        benchmark::BenchRunner::RunAll(args);
 
-    return EXIT_SUCCESS;
+        return EXIT_SUCCESS;
+    } catch (const std::exception& e) {
+        tfm::format(std::cerr, "Error: %s\n", e.what());
+        return EXIT_FAILURE;
+    }
 }

src/bench/block_assemble.cpp

@@ -47,4 +47,4 @@ static void AssembleBlock(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(AssembleBlock);
+BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH);

src/bench/ccoins_caching.cpp

@@ -51,4 +51,4 @@ static void CCoinsCaching(benchmark::Bench& bench)
     ECC_Stop();
 }
 
-BENCHMARK(CCoinsCaching);
+BENCHMARK(CCoinsCaching, benchmark::PriorityLevel::HIGH);

src/bench/chacha20.cpp

@@ -39,6 +39,6 @@ static void CHACHA20_1MB(benchmark::Bench& bench)
     CHACHA20(bench, BUFFER_SIZE_LARGE);
 }
 
-BENCHMARK(CHACHA20_64BYTES);
-BENCHMARK(CHACHA20_256BYTES);
-BENCHMARK(CHACHA20_1MB);
+BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH);

src/bench/chacha_poly_aead.cpp

@@ -115,12 +115,12 @@ static void HASH_1MB(benchmark::Bench& bench)
     HASH(bench, BUFFER_SIZE_LARGE);
 }
 
-BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT);
-BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT);
-BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT);
-BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT);
-BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT);
-BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT);
-BENCHMARK(HASH_64BYTES);
-BENCHMARK(HASH_256BYTES);
-BENCHMARK(HASH_1MB);
+BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH);
+BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH);
+BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH);
+BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH);
+BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH);

src/bench/checkblock.cpp

@@ -50,5 +50,5 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(DeserializeBlockTest);
-BENCHMARK(DeserializeAndCheckBlockTest);
+BENCHMARK(DeserializeBlockTest, benchmark::PriorityLevel::HIGH);
+BENCHMARK(DeserializeAndCheckBlockTest, benchmark::PriorityLevel::HIGH);

src/bench/checkqueue.cpp

@@ -70,4 +70,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench)
     queue.StopWorkerThreads();
     ECC_Stop();
 }
-BENCHMARK(CCheckQueueSpeedPrevectorJob);
+BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH);

src/bench/coin_selection.cpp

@@ -121,5 +121,5 @@ static void BnBExhaustion(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(CoinSelection);
-BENCHMARK(BnBExhaustion);
+BENCHMARK(CoinSelection, benchmark::PriorityLevel::HIGH);
+BENCHMARK(BnBExhaustion, benchmark::PriorityLevel::HIGH);

src/bench/crypto_hash.cpp

@@ -150,19 +150,19 @@ static void MuHashPrecompute(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(RIPEMD160);
-BENCHMARK(SHA1);
-BENCHMARK(SHA256);
-BENCHMARK(SHA512);
-BENCHMARK(SHA3_256_1M);
-
-BENCHMARK(SHA256_32b);
-BENCHMARK(SipHash_32b);
-BENCHMARK(SHA256D64_1024);
-BENCHMARK(FastRandom_32bit);
-BENCHMARK(FastRandom_1bit);
-
-BENCHMARK(MuHash);
-BENCHMARK(MuHashMul);
-BENCHMARK(MuHashDiv);
-BENCHMARK(MuHashPrecompute);
+BENCHMARK(RIPEMD160, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SHA3_256_1M, benchmark::PriorityLevel::HIGH);
+
+BENCHMARK(SHA256_32b, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SipHash_32b, benchmark::PriorityLevel::HIGH);
+BENCHMARK(SHA256D64_1024, benchmark::PriorityLevel::HIGH);
+BENCHMARK(FastRandom_32bit, benchmark::PriorityLevel::HIGH);
+BENCHMARK(FastRandom_1bit, benchmark::PriorityLevel::HIGH);
+
+BENCHMARK(MuHash, benchmark::PriorityLevel::HIGH);
+BENCHMARK(MuHashMul, benchmark::PriorityLevel::HIGH);
+BENCHMARK(MuHashDiv, benchmark::PriorityLevel::HIGH);
+BENCHMARK(MuHashPrecompute, benchmark::PriorityLevel::HIGH);

src/bench/descriptors.cpp

@@ -27,4 +27,4 @@ static void ExpandDescriptor(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(ExpandDescriptor);
+BENCHMARK(ExpandDescriptor, benchmark::PriorityLevel::HIGH);

src/bench/duplicate_inputs.cpp

@@ -62,4 +62,4 @@ static void DuplicateInputs(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(DuplicateInputs);
+BENCHMARK(DuplicateInputs, benchmark::PriorityLevel::HIGH);

src/bench/examples.cpp

@@ -18,4 +18,4 @@ static void Trig(benchmark::Bench& bench)
     });
 }
 
-BENCHMARK(Trig);
+BENCHMARK(Trig, benchmark::PriorityLevel::HIGH);

src/bench/gcs_filter.cpp

@@ -81,8 +81,8 @@ static void GCSFilterMatch(benchmark::Bench& bench)
         filter.Match(GCSFilter::Element());
     });
 }
-BENCHMARK(GCSBlockFilterGetHash);
-BENCHMARK(GCSFilterConstruct);
-BENCHMARK(GCSFilterDecode);
-BENCHMARK(GCSFilterDecodeSkipCheck);
-BENCHMARK(GCSFilterMatch);
+BENCHMARK(GCSBlockFilterGetHash, benchmark::PriorityLevel::HIGH);
+BENCHMARK(GCSFilterConstruct, benchmark::PriorityLevel::HIGH);
+BENCHMARK(GCSFilterDecode, benchmark::PriorityLevel::HIGH);
+BENCHMARK(GCSFilterDecodeSkipCheck, benchmark::PriorityLevel::HIGH);
+BENCHMARK(GCSFilterMatch, benchmark::PriorityLevel::HIGH);