Skip to content

Commit cf4ce1d

Browse files
GuillaumeLagrangeGuillaumeLagrange
committed
feat: add --allow-empty run option
Mostly aimed for action runner CI which just tests cli flags it passes and does not run benches
1 parent f80fb76 commit cf4ce1d

File tree

5 files changed

+69
-16
lines changed

5 files changed

+69
-16
lines changed

‎src/app.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ pub struct Cli {
6060
#[derive(Subcommand, Debug)]
6161
enum Commands {
6262
/// Run the bench command and upload the results to CodSpeed
63-
Run(run::RunArgs),
63+
Run(Box<run::RunArgs>),
6464
/// Manage the CLI authentication state
6565
Auth(auth::AuthArgs),
6666
/// Pre-install the codspeed executors
@@ -88,7 +88,7 @@ pub async fn run() -> Result<()> {
8888

8989
match cli.command {
9090
Commands::Run(args) => {
91-
run::run(args, &api_client, &codspeed_config, setup_cache_dir).await?
91+
run::run(*args, &api_client, &codspeed_config, setup_cache_dir).await?
9292
}
9393
Commands::Auth(args) => auth::run(args, &api_client, cli.config_name.as_deref()).await?,
9494
Commands::Setup => setup::setup(setup_cache_dir).await?,

‎src/run/config.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ pub struct Config {
2525
pub skip_upload: bool,
2626
pub skip_run: bool,
2727
pub skip_setup: bool,
28+
pub allow_empty: bool,
2829
}
2930

3031
#[derive(Debug, PartialEq, Clone)]
@@ -58,6 +59,7 @@ impl Config {
5859
skip_upload: false,
5960
skip_run: false,
6061
skip_setup: false,
62+
allow_empty: false,
6163
}
6264
}
6365
}
@@ -97,6 +99,7 @@ impl TryFrom<RunArgs> for Config {
9799
skip_upload: args.skip_upload,
98100
skip_run: args.skip_run,
99101
skip_setup: args.skip_setup,
102+
allow_empty: args.allow_empty,
100103
})
101104
}
102105
}
@@ -131,6 +134,7 @@ mod tests {
131134
skip_upload: false,
132135
skip_run: false,
133136
skip_setup: false,
137+
allow_empty: false,
134138
perf_run_args: PerfRunArgs {
135139
enable_perf: false,
136140
perf_unwinding_mode: None,
@@ -146,6 +150,7 @@ mod tests {
146150
assert!(!config.skip_upload);
147151
assert!(!config.skip_run);
148152
assert!(!config.skip_setup);
153+
assert!(!config.allow_empty);
149154
assert_eq!(config.command, "cargo codspeed bench");
150155
}
151156

@@ -165,6 +170,7 @@ mod tests {
165170
skip_upload: true,
166171
skip_run: true,
167172
skip_setup: true,
173+
allow_empty: true,
168174
perf_run_args: PerfRunArgs {
169175
enable_perf: false,
170176
perf_unwinding_mode: Some(UnwindingMode::FramePointer),
@@ -199,6 +205,7 @@ mod tests {
199205
assert!(config.skip_upload);
200206
assert!(config.skip_run);
201207
assert!(config.skip_setup);
208+
assert!(config.allow_empty);
202209
assert_eq!(config.command, "cargo codspeed bench");
203210
}
204211

‎src/run/mod.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,10 @@ pub struct RunArgs {
130130
#[arg(long, default_value = "false", hide = true)]
131131
pub skip_setup: bool,
132132

133+
/// Allow runs without any benchmarks to succeed instead of failing
134+
#[arg(long, default_value = "false", hide = true)]
135+
pub allow_empty: bool,
136+
133137
#[command(flatten)]
134138
pub perf_run_args: PerfRunArgs,
135139

@@ -169,6 +173,7 @@ impl RunArgs {
169173
skip_upload: false,
170174
skip_run: false,
171175
skip_setup: false,
176+
allow_empty: false,
172177
perf_run_args: PerfRunArgs {
173178
enable_perf: false,
174179
perf_unwinding_mode: None,

‎src/run/runner/wall_time/executor.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -225,7 +225,7 @@ impl Executor for WallTimeExecutor {
225225
perf.save_files_to(&run_data.profile_folder).await?;
226226
}
227227

228-
validate_walltime_results(&run_data.profile_folder)?;
228+
validate_walltime_results(&run_data.profile_folder, config.allow_empty)?;
229229

230230
Ok(())
231231
}

‎src/run/runner/wall_time/helpers.rs‎

Lines changed: 54 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,15 @@ fn add_empty_result_error_explanation(error_details: &str) -> String {
1111
}
1212

1313
/// Validates that walltime results exist and contain at least one benchmark.
14-
pub fn validate_walltime_results(profile_folder: &Path) -> Result<()> {
14+
/// When `allow_empty` is true, empty benchmark results are allowed.
15+
pub fn validate_walltime_results(profile_folder: &Path, allow_empty: bool) -> Result<()> {
1516
let results_dir = profile_folder.join("results");
1617

1718
if !results_dir.exists() {
19+
if allow_empty {
20+
warn!("No walltime results found in profile folder: {results_dir:?}.");
21+
return Ok(());
22+
}
1823
bail!(add_empty_result_error_explanation(&format!(
1924
"No walltime results found in profile folder: {results_dir:?}."
2025
)));
@@ -41,9 +46,12 @@ pub fn validate_walltime_results(profile_folder: &Path) -> Result<()> {
4146
.with_context(|| format!("Failed to parse walltime results from: {path:?}"))?;
4247

4348
if results.benchmarks.is_empty() {
44-
bail!(add_empty_result_error_explanation(&format!(
45-
"No benchmarks found in walltime results file: {path:?}."
46-
)));
49+
if !allow_empty {
50+
bail!(add_empty_result_error_explanation(&format!(
51+
"No benchmarks found in walltime results file: {path:?}."
52+
)));
53+
}
54+
debug!("No benchmarks found in {path:?} (allowed)");
4755
}
4856

4957
found_valid_results = true;
@@ -54,6 +62,10 @@ pub fn validate_walltime_results(profile_folder: &Path) -> Result<()> {
5462
}
5563

5664
if !found_valid_results {
65+
if allow_empty {
66+
warn!("No JSON result files found in: {results_dir:?}.");
67+
return Ok(());
68+
}
5769
bail!(add_empty_result_error_explanation(&format!(
5870
"No JSON result files found in: {results_dir:?}."
5971
)));
@@ -174,7 +186,7 @@ mod tests {
174186
let profile = TestProfileFolder::new();
175187
profile.write_json_file("results.json", &valid_walltime_results_json(1));
176188

177-
let result = validate_walltime_results(profile.path());
189+
let result = validate_walltime_results(profile.path(), false);
178190
assert!(result.is_ok());
179191
}
180192

@@ -184,7 +196,7 @@ mod tests {
184196
profile.write_json_file("results1.json", &valid_walltime_results_json(2));
185197
profile.write_json_file("results2.json", &valid_walltime_results_json(3));
186198

187-
let result = validate_walltime_results(profile.path());
199+
let result = validate_walltime_results(profile.path(), false);
188200
assert!(result.is_ok());
189201
}
190202

@@ -195,7 +207,7 @@ mod tests {
195207
profile.write_text_file("readme.txt", "This is a text file");
196208
profile.write_text_file("data.csv", "col1,col2");
197209

198-
let result = validate_walltime_results(profile.path());
210+
let result = validate_walltime_results(profile.path(), false);
199211
assert!(result.is_ok());
200212
}
201213

@@ -206,7 +218,7 @@ mod tests {
206218
let profile = TestProfileFolder::new();
207219
// Don't create results directory
208220

209-
let result = validate_walltime_results(profile.path());
221+
let result = validate_walltime_results(profile.path(), false);
210222
assert!(result.is_err());
211223
let error = result.unwrap_err().to_string();
212224
assert!(error.contains("No walltime results found in profile folder"));
@@ -217,7 +229,7 @@ mod tests {
217229
let profile = TestProfileFolder::new();
218230
profile.create_results_dir();
219231

220-
let result = validate_walltime_results(profile.path());
232+
let result = validate_walltime_results(profile.path(), false);
221233
assert!(result.is_err());
222234
let error = result.unwrap_err().to_string();
223235
assert!(error.contains("No JSON result files found in"));
@@ -229,7 +241,7 @@ mod tests {
229241
profile.write_text_file("readme.txt", "some text");
230242
profile.write_text_file("data.csv", "col1,col2");
231243

232-
let result = validate_walltime_results(profile.path());
244+
let result = validate_walltime_results(profile.path(), false);
233245
assert!(result.is_err());
234246
let error = result.unwrap_err().to_string();
235247
assert!(error.contains("No JSON result files found in"));
@@ -240,7 +252,7 @@ mod tests {
240252
let profile = TestProfileFolder::new();
241253
profile.write_json_file("results.json", &empty_benchmarks_json());
242254

243-
let result = validate_walltime_results(profile.path());
255+
let result = validate_walltime_results(profile.path(), false);
244256
assert!(result.is_err());
245257
let error = result.unwrap_err().to_string();
246258
assert!(error.contains("No benchmarks found in walltime results file"));
@@ -251,7 +263,7 @@ mod tests {
251263
let profile = TestProfileFolder::new();
252264
profile.write_json_file("results.json", "{ invalid json }");
253265

254-
let result = validate_walltime_results(profile.path());
266+
let result = validate_walltime_results(profile.path(), false);
255267
assert!(result.is_err());
256268
let error = result.unwrap_err().to_string();
257269
assert!(error.contains("Failed to parse walltime results from"));
@@ -263,9 +275,38 @@ mod tests {
263275
profile.write_json_file("results1.json", &valid_walltime_results_json(2));
264276
profile.write_json_file("results2.json", &empty_benchmarks_json());
265277

266-
let result = validate_walltime_results(profile.path());
278+
let result = validate_walltime_results(profile.path(), false);
267279
assert!(result.is_err());
268280
let error = result.unwrap_err().to_string();
269281
assert!(error.contains("No benchmarks found in walltime results file"));
270282
}
283+
284+
// Allow empty cases
285+
286+
#[test]
287+
fn test_allow_empty_with_empty_benchmarks() {
288+
let profile = TestProfileFolder::new();
289+
profile.write_json_file("results.json", &empty_benchmarks_json());
290+
291+
let result = validate_walltime_results(profile.path(), true);
292+
assert!(result.is_ok());
293+
}
294+
295+
#[test]
296+
fn test_allow_empty_with_missing_results_directory() {
297+
let profile = TestProfileFolder::new();
298+
// Don't create results directory
299+
300+
let result = validate_walltime_results(profile.path(), true);
301+
assert!(result.is_ok());
302+
}
303+
304+
#[test]
305+
fn test_allow_empty_with_no_json_files() {
306+
let profile = TestProfileFolder::new();
307+
profile.create_results_dir();
308+
309+
let result = validate_walltime_results(profile.path(), true);
310+
assert!(result.is_ok());
311+
}
271312
}

0 commit comments

Comments
 (0)