Skip to content

Commit 71187bd

Browse files
not-matthiasnot-matthias
committed
wip: add criterion multiple rounds poc
1 parent 1545735 commit 71187bd

File tree

4 files changed

+223
-89
lines changed

4 files changed

+223
-89
lines changed

‎crates/codspeed/src/compat_utils.rs‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
use crate::codspeed::CodSpeed;
2+
use crate::instrument_hooks::InstrumentHooks;
3+
use std::time::{Duration, Instant};
4+
5+
/// Runs multiple rounds of a benchmark based on CODSPEED_RUNNER_MODE.
6+
///
7+
/// # Important
8+
/// `start_benchmark()` and `end_benchmark()` are called on the OUTSIDE because they
9+
/// clear CPU caches. This is expensive and should only happen once per benchmark.
10+
/// Inside the loop, we use `toggle_collect()` to pause/resume data collection
11+
/// between iterations without clearing caches.
12+
///
13+
/// # Arguments
14+
/// * `codspeed` - The CodSpeed instance to use for benchmarking
15+
/// * `uri` - The benchmark identifier/URI
16+
/// * `run_iteration` - Closure that runs a single benchmark iteration.
17+
/// Should call `toggle_collect()` to resume/pause collection
18+
/// around the measured code.
19+
pub fn run_rounds(codspeed: &mut CodSpeed, uri: &str, mut run_iteration: impl FnMut()) {
20+
let (max_rounds, max_duration) = match std::env::var("CODSPEED_RUNNER_MODE").as_deref() {
21+
Ok("simulation") | Ok("instrumentation") => (None, Some(Duration::from_millis(100))),
22+
Ok("memory") => (Some(1), None),
23+
Ok(m) => unreachable!("Invalid runner mode: {m}"),
24+
Err(err) => panic!("Failed to get runner mode: {err}"),
25+
};
26+
27+
let mut rounds = 0;
28+
let rounds_start_time = Instant::now();
29+
30+
// Start benchmark ONCE - this clears CPU caches
31+
codspeed.start_benchmark(uri);
32+
InstrumentHooks::toggle_collect(); // Pause collection before first iteration
33+
34+
loop {
35+
rounds += 1;
36+
37+
run_iteration();
38+
39+
let within_rounds = max_rounds.map_or(true, |max| rounds < max);
40+
let within_duration = max_duration.map_or(true, |max| rounds_start_time.elapsed() < max);
41+
42+
if !(within_rounds && within_duration) {
43+
break;
44+
}
45+
}
46+
47+
// End benchmark ONCE
48+
codspeed.end_benchmark();
49+
}

‎crates/codspeed/src/lib.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
pub mod codspeed;
2+
pub mod compat_utils;
23

34
pub mod instrument_hooks;
45

‎crates/criterion_compat/src/compat/bencher.rs‎

Lines changed: 170 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
use codspeed::codspeed::{black_box, CodSpeed};
2+
use codspeed::compat_utils;
3+
use codspeed::instrument_hooks::InstrumentHooks;
24
use colored::Colorize;
35
use criterion::BatchSize;
46

@@ -25,15 +27,19 @@ impl<'a> Bencher<'a> {
2527
{
2628
// NOTE: this structure hardens our benchmark against dead code elimination
2729
// https://godbolt.org/z/KnYeKMd1o
28-
for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 {
29-
if i < codspeed::codspeed::WARMUP_RUNS {
30-
black_box(routine());
31-
} else {
32-
self.codspeed.start_benchmark(self.uri.as_str());
33-
black_box(routine());
34-
self.codspeed.end_benchmark();
35-
}
30+
31+
// Warmup runs
32+
for _ in 0..codspeed::codspeed::WARMUP_RUNS {
33+
black_box(routine());
3634
}
35+
36+
// Multiple measured rounds
37+
compat_utils::run_rounds(self.codspeed, self.uri.as_str(), || {
38+
InstrumentHooks::toggle_collect(); // Resume collection
39+
let output = routine();
40+
InstrumentHooks::toggle_collect(); // Pause collection
41+
black_box(output);
42+
});
3743
}
3844

3945
#[inline(never)]
@@ -54,19 +60,21 @@ impl<'a> Bencher<'a> {
5460
S: FnMut() -> I,
5561
R: FnMut(I) -> O,
5662
{
57-
for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 {
63+
// Warmup runs
64+
for _ in 0..codspeed::codspeed::WARMUP_RUNS {
5865
let input = black_box(setup());
59-
let output = if i < codspeed::codspeed::WARMUP_RUNS {
60-
routine(input)
61-
} else {
62-
let input = black_box(setup());
63-
self.codspeed.start_benchmark(self.uri.as_str());
64-
let output = routine(input);
65-
self.codspeed.end_benchmark();
66-
output
67-
};
66+
let output = routine(input);
6867
drop(black_box(output));
6968
}
69+
70+
// Multiple measured rounds
71+
compat_utils::run_rounds(self.codspeed, self.uri.as_str(), || {
72+
let input = setup(); // Setup runs while collection is paused
73+
InstrumentHooks::toggle_collect(); // Resume collection
74+
let output = routine(input);
75+
InstrumentHooks::toggle_collect(); // Pause collection
76+
black_box(output);
77+
});
7078
}
7179

7280
pub fn iter_with_setup<I, O, S, R>(&mut self, setup: S, routine: R)
@@ -98,19 +106,23 @@ impl<'a> Bencher<'a> {
98106
S: FnMut() -> I,
99107
R: FnMut(&mut I) -> O,
100108
{
101-
for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 {
109+
// Warmup runs
110+
for _ in 0..codspeed::codspeed::WARMUP_RUNS {
102111
let mut input = black_box(setup());
103-
let output = if i < codspeed::codspeed::WARMUP_RUNS {
104-
black_box(routine(&mut input))
105-
} else {
106-
self.codspeed.start_benchmark(self.uri.as_str());
107-
let output = black_box(routine(&mut input));
108-
self.codspeed.end_benchmark();
109-
output
110-
};
112+
let output = black_box(routine(&mut input));
111113
drop(black_box(output));
112114
drop(black_box(input));
113115
}
116+
117+
// Multiple measured rounds
118+
compat_utils::run_rounds(self.codspeed, self.uri.as_str(), || {
119+
let mut input = setup(); // Setup runs while collection is paused
120+
InstrumentHooks::toggle_collect(); // Resume collection
121+
let output = routine(&mut input);
122+
InstrumentHooks::toggle_collect(); // Pause collection
123+
black_box(input);
124+
black_box(output);
125+
});
114126
}
115127

116128
#[cfg(feature = "async")]
@@ -135,17 +147,52 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> {
135147
R: FnMut() -> F,
136148
F: Future<Output = O>,
137149
{
150+
use std::time::{Duration, Instant};
151+
138152
let AsyncBencher { b, runner } = self;
139153
runner.block_on(async {
140-
for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 {
141-
if i < codspeed::codspeed::WARMUP_RUNS {
142-
black_box(routine().await);
143-
} else {
144-
b.codspeed.start_benchmark(b.uri.as_str());
145-
black_box(routine().await);
146-
b.codspeed.end_benchmark();
154+
// Warmup runs
155+
for _ in 0..codspeed::codspeed::WARMUP_RUNS {
156+
black_box(routine().await);
157+
}
158+
159+
// Multiple measured rounds
160+
let (max_rounds, max_duration) = match std::env::var("CODSPEED_RUNNER_MODE").as_deref()
161+
{
162+
Ok("simulation") | Ok("instrumentation") => {
163+
(None, Some(Duration::from_millis(100)))
164+
}
165+
Ok("memory") => (Some(1), None),
166+
Ok(m) => unreachable!("Invalid runner mode: {m}"),
167+
Err(err) => panic!("Failed to get runner mode: {err}"),
168+
};
169+
170+
let mut rounds = 0;
171+
let rounds_start_time = Instant::now();
172+
173+
// Start benchmark ONCE - this clears CPU caches
174+
b.codspeed.start_benchmark(b.uri.as_str());
175+
InstrumentHooks::toggle_collect(); // Pause collection before first iteration
176+
177+
loop {
178+
rounds += 1;
179+
180+
InstrumentHooks::toggle_collect(); // Resume collection
181+
let output = routine().await;
182+
InstrumentHooks::toggle_collect(); // Pause collection
183+
black_box(output);
184+
185+
let within_rounds = max_rounds.map_or(true, |max| rounds < max);
186+
let within_duration =
187+
max_duration.map_or(true, |max| rounds_start_time.elapsed() < max);
188+
189+
if !(within_rounds && within_duration) {
190+
break;
147191
}
148192
}
193+
194+
// End benchmark ONCE
195+
b.codspeed.end_benchmark();
149196
});
150197
}
151198

@@ -199,20 +246,55 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> {
199246
R: FnMut(I) -> F,
200247
F: Future<Output = O>,
201248
{
249+
use std::time::{Duration, Instant};
250+
202251
let AsyncBencher { b, runner } = self;
203252
runner.block_on(async {
204-
for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 {
253+
// Warmup runs
254+
for _ in 0..codspeed::codspeed::WARMUP_RUNS {
205255
let input = black_box(setup());
206-
let output = if i < codspeed::codspeed::WARMUP_RUNS {
207-
routine(input).await
208-
} else {
209-
b.codspeed.start_benchmark(b.uri.as_str());
210-
let output = routine(input).await;
211-
b.codspeed.end_benchmark();
212-
output
213-
};
256+
let output = routine(input).await;
214257
drop(black_box(output));
215258
}
259+
260+
// Multiple measured rounds
261+
let (max_rounds, max_duration) = match std::env::var("CODSPEED_RUNNER_MODE").as_deref()
262+
{
263+
Ok("simulation") | Ok("instrumentation") => {
264+
(None, Some(Duration::from_millis(100)))
265+
}
266+
Ok("memory") => (Some(1), None),
267+
Ok(m) => unreachable!("Invalid runner mode: {m}"),
268+
Err(err) => panic!("Failed to get runner mode: {err}"),
269+
};
270+
271+
let mut rounds = 0;
272+
let rounds_start_time = Instant::now();
273+
274+
// Start benchmark ONCE - this clears CPU caches
275+
b.codspeed.start_benchmark(b.uri.as_str());
276+
InstrumentHooks::toggle_collect(); // Pause collection before first iteration
277+
278+
loop {
279+
rounds += 1;
280+
281+
let input = setup(); // Setup runs while collection is paused
282+
InstrumentHooks::toggle_collect(); // Resume collection
283+
let output = routine(input).await;
284+
InstrumentHooks::toggle_collect(); // Pause collection
285+
black_box(output);
286+
287+
let within_rounds = max_rounds.map_or(true, |max| rounds < max);
288+
let within_duration =
289+
max_duration.map_or(true, |max| rounds_start_time.elapsed() < max);
290+
291+
if !(within_rounds && within_duration) {
292+
break;
293+
}
294+
}
295+
296+
// End benchmark ONCE
297+
b.codspeed.end_benchmark();
216298
})
217299
}
218300

@@ -228,21 +310,57 @@ impl<'a, 'b, A: AsyncExecutor> AsyncBencher<'a, 'b, A> {
228310
R: FnMut(&mut I) -> F,
229311
F: Future<Output = O>,
230312
{
313+
use std::time::{Duration, Instant};
314+
231315
let AsyncBencher { b, runner } = self;
232316
runner.block_on(async {
233-
for i in 0..codspeed::codspeed::WARMUP_RUNS + 1 {
317+
// Warmup runs
318+
for _ in 0..codspeed::codspeed::WARMUP_RUNS {
234319
let mut input = black_box(setup());
235-
let output = if i < codspeed::codspeed::WARMUP_RUNS {
236-
black_box(routine(&mut input).await)
237-
} else {
238-
b.codspeed.start_benchmark(b.uri.as_str());
239-
let output = black_box(routine(&mut input).await);
240-
b.codspeed.end_benchmark();
241-
output
242-
};
320+
let output = black_box(routine(&mut input).await);
243321
drop(black_box(output));
244322
drop(black_box(input));
245323
}
324+
325+
// Multiple measured rounds
326+
let (max_rounds, max_duration) = match std::env::var("CODSPEED_RUNNER_MODE").as_deref()
327+
{
328+
Ok("simulation") | Ok("instrumentation") => {
329+
(None, Some(Duration::from_millis(100)))
330+
}
331+
Ok("memory") => (Some(1), None),
332+
Ok(m) => unreachable!("Invalid runner mode: {m}"),
333+
Err(err) => panic!("Failed to get runner mode: {err}"),
334+
};
335+
336+
let mut rounds = 0;
337+
let rounds_start_time = Instant::now();
338+
339+
// Start benchmark ONCE - this clears CPU caches
340+
b.codspeed.start_benchmark(b.uri.as_str());
341+
InstrumentHooks::toggle_collect(); // Pause collection before first iteration
342+
343+
loop {
344+
rounds += 1;
345+
346+
let mut input = setup(); // Setup runs while collection is paused
347+
InstrumentHooks::toggle_collect(); // Resume collection
348+
let output = routine(&mut input).await;
349+
InstrumentHooks::toggle_collect(); // Pause collection
350+
black_box(input);
351+
black_box(output);
352+
353+
let within_rounds = max_rounds.map_or(true, |max| rounds < max);
354+
let within_duration =
355+
max_duration.map_or(true, |max| rounds_start_time.elapsed() < max);
356+
357+
if !(within_rounds && within_duration) {
358+
break;
359+
}
360+
}
361+
362+
// End benchmark ONCE
363+
b.codspeed.end_benchmark();
246364
});
247365
}
248366
}

0 commit comments

Comments
 (0)