Skip to content

Commit ed5316b

Browse files
filzrevtimcassell
filzrev
and
timcassell
authored
feat: Add validator for benchmarks that contains null runtime (#2771)
* feat: add validator for job that contains null runtime * chore: update validator logics * Update src/BenchmarkDotNet/Validators/RuntimeValidator.cs Co-authored-by: Tim Cassell <cassell.timothy@gmail.com> * chore: fix test validation messages --------- Co-authored-by: Tim Cassell <cassell.timothy@gmail.com>
1 parent de8ba07 commit ed5316b

File tree

3 files changed

+169
-0
lines changed

3 files changed

+169
-0
lines changed

‎src/BenchmarkDotNet/Configs/DefaultConfig.cs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ public IEnumerable<IValidator> GetValidators()
7272
yield return DeferredExecutionValidator.FailOnError;
7373
yield return ParamsAllValuesValidator.FailOnError;
7474
yield return ParamsValidator.FailOnError;
75+
yield return RuntimeValidator.DontFailOnError;
7576
}
7677

7778
public IOrderer Orderer => null;

‎src/BenchmarkDotNet/Validators/RuntimeValidator.cs

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
using System.Collections.Generic;
2+
using System.Linq;
3+
using BenchmarkDotNet.Characteristics;
4+
5+
namespace BenchmarkDotNet.Validators;
6+
7+
/// <summary>
8+
/// Validator for runtime characteristic.
9+
/// <see href="https://github.com/dotnet/BenchmarkDotNet/issues/2609" />
10+
/// </summary>
11+
public class RuntimeValidator : IValidator
12+
{
13+
public static readonly IValidator DontFailOnError = new RuntimeValidator();
14+
15+
private RuntimeValidator() { }
16+
17+
public bool TreatsWarningsAsErrors => false;
18+
19+
public IEnumerable<ValidationError> Validate(ValidationParameters input)
20+
{
21+
var allBenchmarks = input.Benchmarks.ToArray();
22+
var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray();
23+
24+
// There is no validation error if all the runtimes are set or if all the runtimes are null.
25+
if (allBenchmarks.Length == nullRuntimeBenchmarks.Length)
26+
{
27+
return [];
28+
}
29+
30+
var errors = new List<ValidationError>();
31+
foreach (var benchmark in nullRuntimeBenchmarks)
32+
{
33+
var job = benchmark.Job;
34+
var jobText = job.HasValue(CharacteristicObject.IdCharacteristic)
35+
? job.Id
36+
: CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId.
37+
38+
var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly.";
39+
errors.Add(new ValidationError(false, message));
40+
}
41+
return errors;
42+
}
43+
}

‎tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
using BenchmarkDotNet.Attributes;
2+
using BenchmarkDotNet.Configs;
3+
using BenchmarkDotNet.Environments;
4+
using BenchmarkDotNet.Jobs;
5+
using BenchmarkDotNet.Running;
6+
using BenchmarkDotNet.Toolchains.CsProj;
7+
using BenchmarkDotNet.Validators;
8+
using System.Linq;
9+
using Xunit;
10+
11+
namespace BenchmarkDotNet.Tests.Validators;
12+
13+
public class RuntimeValidatorTests
14+
{
15+
[Fact]
16+
public void SameRuntime_Should_Success()
17+
{
18+
// Arrange
19+
var config = new TestConfig1().CreateImmutableConfig();
20+
var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config);
21+
var parameters = new ValidationParameters(runInfo.BenchmarksCases, config);
22+
23+
// Act
24+
var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray();
25+
26+
// Assert
27+
Assert.Empty(errors);
28+
}
29+
30+
[Fact]
31+
public void NullRuntimeMixed_Should_Failed()
32+
{
33+
// Arrange
34+
var config = new TestConfig2().CreateImmutableConfig();
35+
var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config);
36+
var parameters = new ValidationParameters(runInfo.BenchmarksCases, config);
37+
38+
// Act
39+
var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray();
40+
41+
// Assert
42+
{
43+
var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly.";
44+
Assert.Contains(expectedMessage, errors);
45+
}
46+
{
47+
var expectedMessage = "Job(Toolchain=.NET 10.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly.";
48+
Assert.Contains(expectedMessage, errors);
49+
}
50+
}
51+
52+
[Fact]
53+
public void NotNullRuntimeOnly_Should_Success()
54+
{
55+
// Arrange
56+
var config = new TestConfig3().CreateImmutableConfig();
57+
var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config);
58+
var parameters = new ValidationParameters(runInfo.BenchmarksCases, config);
59+
60+
// Act
61+
var errors = RuntimeValidator.DontFailOnError.Validate(parameters).Select(e => e.Message).ToArray();
62+
63+
// Assert
64+
Assert.Empty(errors);
65+
}
66+
67+
public class DummyBenchmark
68+
{
69+
[Benchmark]
70+
public void Benchmark()
71+
{
72+
}
73+
}
74+
75+
// TestConfig that expicitly specify runtime.
76+
private class TestConfig1 : ManualConfig
77+
{
78+
public TestConfig1()
79+
{
80+
var baseJob = Job.Dry;
81+
82+
WithOption(ConfigOptions.DisableOptimizationsValidator, true);
83+
AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray());
84+
85+
AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80));
86+
AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90));
87+
}
88+
}
89+
90+
// TestConfig that contains job that don't specify runtime.
91+
private class TestConfig2 : ManualConfig
92+
{
93+
public TestConfig2()
94+
{
95+
var baseJob = Job.Dry;
96+
97+
WithOption(ConfigOptions.DisableOptimizationsValidator, true);
98+
AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray());
99+
100+
AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80));
101+
AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)
102+
.WithRuntime(CoreRuntime.Core90));
103+
104+
// Validate error message for auto generated jobid.
105+
AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0));
106+
}
107+
}
108+
109+
// TestConfig that expicitly specify runtime.
110+
private class TestConfig3 : ManualConfig
111+
{
112+
public TestConfig3()
113+
{
114+
var baseJob = Job.Dry;
115+
116+
WithOption(ConfigOptions.DisableOptimizationsValidator, true);
117+
AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray());
118+
119+
AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)
120+
.WithRuntime(CoreRuntime.Core80)); ;
121+
AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)
122+
.WithRuntime(CoreRuntime.Core90));
123+
}
124+
}
125+
}

0 commit comments

Comments
 (0)