Merge pull request #1638 from Kobzol/runtime-benchmarks-changes

Runtime benchmarks changes

Author: Kobzol

collector/benchlib/src/benchmark.rs

@@ -80,6 +80,13 @@ impl BenchmarkGroup {
 
         for (name, benchmark_fn) in items {
             let mut stats: Vec<BenchmarkStats> = Vec::with_capacity(args.iterations as usize);
+            // Warm-up
+            for _ in 0..3 {
+                let benchmark_stats = benchmark_fn()?;
+                black_box(benchmark_stats);
+            }
+
+            // Actual measurement
             for i in 0..args.iterations {
                 let benchmark_stats = benchmark_fn()?;
                 log::info!("Benchmark (run {i}) `{name}` completed: {benchmark_stats:?}");
@@ -120,7 +127,7 @@ pub fn passes_filter(name: &str, exclude: Option<&str>, include: Option<&str>) -
     }
 }
 
-/// Copied from `iai`, so that we don't have to use unstable features.
+/// Copied from `iai`, so that it works on Rustc older than 1.66.
 pub fn black_box<T>(dummy: T) -> T {
     unsafe {
         let ret = std::ptr::read_volatile(&dummy);

collector/runtime-benchmarks/bufreader/Cargo.lock

@@ -1,5 +1,5 @@
 [package]
-name = "bufreader"
+name = "bufreader-bench"
 version = "0.1.0"
 edition = "2021"
 

collector/runtime-benchmarks/bufreader/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 benchlib = { path = "../../benchlib" }
-hashbrown = "0.12.3"
+hashbrown = { version = "0.13.2" }
 fxhash = "0.2.1"
 
 [workspace]

collector/runtime-benchmarks/hashmap/Cargo.lock

@@ -1,19 +1,85 @@
 use benchlib;
-use benchlib::benchmark::run_benchmark_group;
+use benchlib::benchmark::{black_box, run_benchmark_group};
 
 fn main() {
     run_benchmark_group(|group| {
-        // Measures how long does it take to insert 10 thousand numbers into a `hashbrown` hashmap.
-        group.register_benchmark("hashmap_insert_10k", || {
+        // Measures how long does it take to insert 1 million numbers into a hashmap.
+        group.register_benchmark("hashmap_insert_1m", || {
+            let count = 1_000_000;
             let mut map = hashbrown::HashMap::with_capacity_and_hasher(
-                10000,
+                // Over allocate the hashmap to avoid reallocations when inserting
+                count * 2,
                 fxhash::FxBuildHasher::default(),
             );
             move || {
-                for index in 0..10000 {
+                for index in 0..count {
                     map.insert(index, index);
                 }
             }
         });
+
+        // Measures how long it takes to remove 1 million elements from a hashmap.
+        group.register_benchmark("hashmap_remove_1m", || {
+            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
+                1_000_000,
+                fxhash::FxBuildHasher::default(),
+            );
+            for index in 0..map.capacity() {
+                map.insert(index, index);
+            }
+
+            move || {
+                for index in 0..map.capacity() {
+                    map.remove(&index);
+                }
+            }
+        });
+
+        // Measures how long it takes to find 1 million elements that are in a hashmap.
+        group.register_benchmark("hashmap_find_1m", || {
+            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
+                1_000_000,
+                fxhash::FxBuildHasher::default(),
+            );
+            for index in 0..map.capacity() {
+                map.insert(index, index);
+            }
+
+            move || {
+                for index in 0..map.capacity() {
+                    black_box(map.get(&index));
+                }
+            }
+        });
+
+        // Measures how long it takes to find 1 million elements that are not in a hashmap.
+        group.register_benchmark("hashmap_find_misses_1m", || {
+            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
+                1_000_000,
+                fxhash::FxBuildHasher::default(),
+            );
+            for index in 0..map.capacity() {
+                map.insert(index, index);
+            }
+
+            move || {
+                for index in map.capacity()..(map.capacity() * 2) {
+                    black_box(map.get(&index));
+                }
+            }
+        });
+
+        // Measures how long it takes to iterate through values of a hashmap with 1 million elements.
+        group.register_benchmark("hashmap_iterate_1m", || {
+            let mut map = hashbrown::HashMap::with_capacity_and_hasher(
+                1_000_000,
+                fxhash::FxBuildHasher::default(),
+            );
+            for index in 0..map.capacity() {
+                map.insert(index, index as u64);
+            }
+
+            move || map.values().sum::<u64>()
+        });
     });
 }

collector/runtime-benchmarks/hashmap/Cargo.toml

@@ -1,19 +1,18 @@
-//! Calculates the N-body simulation.
-//! Code taken from https://github.com/prestontw/rust-nbody
-
 use benchlib::benchmark::run_benchmark_group;
 
 mod nbody;
 
 fn main() {
     run_benchmark_group(|group| {
-        group.register_benchmark("nbody_10k", || {
-            let mut nbody_10k = nbody::init(10000);
+        // Calculates the N-body simulation.
+        // Code taken from https://github.com/prestontw/rust-nbody
+        group.register_benchmark("nbody_5k", || {
+            let mut nbody = nbody::init(5000);
             || {
                 for _ in 0..10 {
-                    nbody_10k = nbody::compute_forces(nbody_10k);
+                    nbody = nbody::compute_forces(nbody);
                 }
-                nbody_10k
+                nbody
             }
         });
     });

collector/runtime-benchmarks/hashmap/src/main.rs

@@ -221,9 +221,11 @@ fn print_stats(result: &BenchmarkResult) {
                     .map(|v| (v as f64 - mean).powf(2.0)),
             )
             .sqrt();
+            let min = result.stats.iter().map(&f).min().unwrap_or(0);
 
             println!(
-                "{name:>18}: {:>16} (+/- {:>11})",
+                "{name:>18}: min:{:>16}    mean: {:>16}    stddev: {:>11}",
+                min.separate_with_commas(),
                 (mean as u64).separate_with_commas(),
                 (stddev as u64).separate_with_commas()
             );