Skip to content

Commit 33f392c

Browse files
Copilottimcassell
Copilot
and
timcassell
committed
Add tests for multiple in-process diagnosers with varying run modes
Co-authored-by: timcassell <35501420+timcassell@users.noreply.github.com>
1 parent 431b88d commit 33f392c

File tree

2 files changed

+297
-0
lines changed

2 files changed

+297
-0
lines changed

‎tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,109 @@ public void Initialize(string? serializedConfig) { }
4747
public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
4848

4949
public string SerializeResults() => "MockResult";
50+
}
51+
52+
public sealed class MockInProcessDiagnoserNoOverhead : IInProcessDiagnoser
53+
{
54+
public Dictionary<BenchmarkCase, string> Results { get; } = [];
55+
56+
public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoserNoOverhead)];
57+
58+
public IEnumerable<IExporter> Exporters => [];
59+
60+
public IEnumerable<IAnalyser> Analysers => [];
61+
62+
public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoserNoOverhead)} results: [{string.Join(", ", Results.Values)}]");
63+
64+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
65+
66+
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
67+
68+
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
69+
70+
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
71+
72+
public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
73+
=> (typeof(MockInProcessDiagnoserNoOverheadHandler), null);
74+
75+
public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
76+
=> new MockInProcessDiagnoserNoOverheadHandler();
77+
78+
public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
79+
}
80+
81+
public sealed class MockInProcessDiagnoserNoOverheadHandler : IInProcessDiagnoserHandler
82+
{
83+
public void Initialize(string? serializedConfig) { }
84+
85+
public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
86+
87+
public string SerializeResults() => "NoOverheadResult";
88+
}
89+
90+
public sealed class MockInProcessDiagnoserExtraRun : IInProcessDiagnoser
91+
{
92+
public Dictionary<BenchmarkCase, string> Results { get; } = [];
93+
94+
public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoserExtraRun)];
95+
96+
public IEnumerable<IExporter> Exporters => [];
97+
98+
public IEnumerable<IAnalyser> Analysers => [];
99+
100+
public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoserExtraRun)} results: [{string.Join(", ", Results.Values)}]");
101+
102+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.ExtraRun;
103+
104+
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
105+
106+
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
107+
108+
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
109+
110+
public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
111+
=> (typeof(MockInProcessDiagnoserExtraRunHandler), null);
112+
113+
public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
114+
=> new MockInProcessDiagnoserExtraRunHandler();
115+
116+
public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
117+
}
118+
119+
public sealed class MockInProcessDiagnoserExtraRunHandler : IInProcessDiagnoserHandler
120+
{
121+
public void Initialize(string? serializedConfig) { }
122+
123+
public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
124+
125+
public string SerializeResults() => "ExtraRunResult";
126+
}
127+
128+
public sealed class MockInProcessDiagnoserNone : IInProcessDiagnoser
129+
{
130+
public Dictionary<BenchmarkCase, string> Results { get; } = [];
131+
132+
public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoserNone)];
133+
134+
public IEnumerable<IExporter> Exporters => [];
135+
136+
public IEnumerable<IAnalyser> Analysers => [];
137+
138+
public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoserNone)} results: [{string.Join(", ", Results.Values)}]");
139+
140+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.None;
141+
142+
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
143+
144+
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
145+
146+
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
147+
148+
public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
149+
=> default; // Returns default when RunMode is None
150+
151+
public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
152+
=> null; // Returns null when RunMode is None
153+
154+
public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
50155
}

‎tests/BenchmarkDotNet.IntegrationTests/MultipleInProcessDiagnosersTests.cs‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
using System.Linq;
2+
using System.Threading;
3+
using BenchmarkDotNet.Attributes;
4+
using BenchmarkDotNet.Columns;
5+
using BenchmarkDotNet.Configs;
6+
using BenchmarkDotNet.IntegrationTests.Diagnosers;
7+
using BenchmarkDotNet.Jobs;
8+
using BenchmarkDotNet.Tests.Loggers;
9+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
10+
using Xunit;
11+
using Xunit.Abstractions;
12+
13+
namespace BenchmarkDotNet.IntegrationTests;
14+
15+
public class MultipleInProcessDiagnosersTests : BenchmarkTestExecutor
16+
{
17+
public MultipleInProcessDiagnosersTests(ITestOutputHelper output) : base(output) { }
18+
19+
[Fact]
20+
public void MultipleInProcessDiagnosersWithNoOverheadRunMode()
21+
{
22+
var logger = new OutputLogger(Output);
23+
var diagnoser1 = new MockInProcessDiagnoserNoOverhead();
24+
var diagnoser2 = new MockInProcessDiagnoser();
25+
26+
var config = CreateInProcessConfig(logger)
27+
.AddDiagnoser(diagnoser1)
28+
.AddDiagnoser(diagnoser2);
29+
30+
var summary = CanExecute<SimpleBenchmark>(config);
31+
32+
// Both diagnosers should have results for each benchmark
33+
Assert.NotEmpty(diagnoser1.Results);
34+
Assert.NotEmpty(diagnoser2.Results);
35+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser1.Results.Count);
36+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser2.Results.Count);
37+
38+
// Verify the results are correct for each diagnoser
39+
Assert.All(diagnoser1.Results.Values, result => Assert.Equal("NoOverheadResult", result));
40+
Assert.All(diagnoser2.Results.Values, result => Assert.Equal("MockResult", result));
41+
}
42+
43+
[Fact]
44+
public void MultipleInProcessDiagnosersWithExtraRunRunMode()
45+
{
46+
var logger = new OutputLogger(Output);
47+
var diagnoser1 = new MockInProcessDiagnoserExtraRun();
48+
var diagnoser2 = new MockInProcessDiagnoserNoOverhead();
49+
50+
var config = CreateInProcessConfig(logger)
51+
.AddDiagnoser(diagnoser1)
52+
.AddDiagnoser(diagnoser2);
53+
54+
var summary = CanExecute<SimpleBenchmark>(config);
55+
56+
// Both diagnosers should have results
57+
Assert.NotEmpty(diagnoser1.Results);
58+
Assert.NotEmpty(diagnoser2.Results);
59+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser1.Results.Count);
60+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser2.Results.Count);
61+
62+
// Verify the results are correct for each diagnoser
63+
Assert.All(diagnoser1.Results.Values, result => Assert.Equal("ExtraRunResult", result));
64+
Assert.All(diagnoser2.Results.Values, result => Assert.Equal("NoOverheadResult", result));
65+
}
66+
67+
[Fact]
68+
public void MultipleInProcessDiagnosersWithVaryingRunModes()
69+
{
70+
var logger = new OutputLogger(Output);
71+
var noOverheadDiagnoser = new MockInProcessDiagnoserNoOverhead();
72+
var extraRunDiagnoser = new MockInProcessDiagnoserExtraRun();
73+
var noneDiagnoser = new MockInProcessDiagnoserNone();
74+
75+
var config = CreateInProcessConfig(logger)
76+
.AddDiagnoser(noOverheadDiagnoser)
77+
.AddDiagnoser(extraRunDiagnoser)
78+
.AddDiagnoser(noneDiagnoser);
79+
80+
var summary = CanExecute<SimpleBenchmark>(config);
81+
82+
// NoOverhead and ExtraRun diagnosers should have results
83+
Assert.NotEmpty(noOverheadDiagnoser.Results);
84+
Assert.NotEmpty(extraRunDiagnoser.Results);
85+
Assert.Equal(summary.BenchmarksCases.Length, noOverheadDiagnoser.Results.Count);
86+
Assert.Equal(summary.BenchmarksCases.Length, extraRunDiagnoser.Results.Count);
87+
88+
// None diagnoser should not have results (RunMode.None means it shouldn't run)
89+
Assert.Empty(noneDiagnoser.Results);
90+
91+
// Verify the results are correct for diagnosers that ran
92+
Assert.All(noOverheadDiagnoser.Results.Values, result => Assert.Equal("NoOverheadResult", result));
93+
Assert.All(extraRunDiagnoser.Results.Values, result => Assert.Equal("ExtraRunResult", result));
94+
}
95+
96+
[Fact]
97+
public void ThreeDifferentTypesOfInProcessDiagnosers()
98+
{
99+
var logger = new OutputLogger(Output);
100+
var noOverheadDiagnoser = new MockInProcessDiagnoserNoOverhead();
101+
var mockDiagnoser = new MockInProcessDiagnoser();
102+
var extraRunDiagnoser = new MockInProcessDiagnoserExtraRun();
103+
104+
var config = CreateInProcessConfig(logger)
105+
.AddDiagnoser(noOverheadDiagnoser)
106+
.AddDiagnoser(mockDiagnoser)
107+
.AddDiagnoser(extraRunDiagnoser);
108+
109+
var summary = CanExecute<SimpleBenchmark>(config);
110+
111+
// All three diagnosers should have results
112+
Assert.NotEmpty(noOverheadDiagnoser.Results);
113+
Assert.NotEmpty(mockDiagnoser.Results);
114+
Assert.NotEmpty(extraRunDiagnoser.Results);
115+
Assert.Equal(summary.BenchmarksCases.Length, noOverheadDiagnoser.Results.Count);
116+
Assert.Equal(summary.BenchmarksCases.Length, mockDiagnoser.Results.Count);
117+
Assert.Equal(summary.BenchmarksCases.Length, extraRunDiagnoser.Results.Count);
118+
119+
// Verify the results are correct for each diagnoser
120+
Assert.All(noOverheadDiagnoser.Results.Values, result => Assert.Equal("NoOverheadResult", result));
121+
Assert.All(mockDiagnoser.Results.Values, result => Assert.Equal("MockResult", result));
122+
Assert.All(extraRunDiagnoser.Results.Values, result => Assert.Equal("ExtraRunResult", result));
123+
}
124+
125+
[Fact]
126+
public void MultipleInProcessDiagnosersWithMultipleBenchmarks()
127+
{
128+
var logger = new OutputLogger(Output);
129+
var diagnoser1 = new MockInProcessDiagnoserNoOverhead();
130+
var diagnoser2 = new MockInProcessDiagnoserExtraRun();
131+
132+
var config = CreateInProcessConfig(logger)
133+
.AddDiagnoser(diagnoser1)
134+
.AddDiagnoser(diagnoser2);
135+
136+
var summary = CanExecute<MultipleBenchmarks>(config);
137+
138+
// Both diagnosers should have results for all benchmarks
139+
Assert.NotEmpty(diagnoser1.Results);
140+
Assert.NotEmpty(diagnoser2.Results);
141+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser1.Results.Count);
142+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser2.Results.Count);
143+
144+
// Verify each diagnoser has a result for each benchmark method
145+
var benchmarkMethods = summary.BenchmarksCases.Select(bc => bc.Descriptor.WorkloadMethod.Name).ToList();
146+
Assert.Contains("Benchmark1", benchmarkMethods);
147+
Assert.Contains("Benchmark2", benchmarkMethods);
148+
Assert.Contains("Benchmark3", benchmarkMethods);
149+
}
150+
151+
private IConfig CreateInProcessConfig(OutputLogger logger)
152+
{
153+
return new ManualConfig()
154+
.AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.DontLogOutput))
155+
.AddLogger(logger)
156+
.AddColumnProvider(DefaultColumnProviders.Instance);
157+
}
158+
159+
public class SimpleBenchmark
160+
{
161+
private int counter;
162+
163+
[Benchmark]
164+
public void BenchmarkMethod()
165+
{
166+
Interlocked.Increment(ref counter);
167+
}
168+
}
169+
170+
public class MultipleBenchmarks
171+
{
172+
private int counter;
173+
174+
[Benchmark]
175+
public void Benchmark1()
176+
{
177+
Interlocked.Increment(ref counter);
178+
}
179+
180+
[Benchmark]
181+
public void Benchmark2()
182+
{
183+
Interlocked.Increment(ref counter);
184+
}
185+
186+
[Benchmark]
187+
public void Benchmark3()
188+
{
189+
Interlocked.Increment(ref counter);
190+
}
191+
}
192+
}

0 commit comments

Comments
 (0)