Skip to content

Commit 47904ea

Browse files
CopilotCopilot
authored
Add tests for multiple in-process diagnosers with varying run modes (#2857)
Add extensive InProcessDiagnoserTests.
1 parent eb27f66 commit 47904ea

File tree

6 files changed

+251
-41
lines changed

6 files changed

+251
-41
lines changed

‎tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs‎

Lines changed: 95 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -11,40 +11,122 @@
1111

1212
namespace BenchmarkDotNet.IntegrationTests.Diagnosers;
1313

14-
public sealed class MockInProcessDiagnoser : IInProcessDiagnoser
14+
public abstract class BaseMockInProcessDiagnoser : IInProcessDiagnoser
1515
{
16+
public static Queue<BaseMockInProcessDiagnoser> s_completedResults = new();
17+
1618
public Dictionary<BenchmarkCase, string> Results { get; } = [];
1719

18-
public IEnumerable<string> Ids => [nameof(MockInProcessDiagnoser)];
20+
public abstract string DiagnoserName { get; }
21+
public abstract RunMode DiagnoserRunMode { get; }
22+
public abstract string ExpectedResult { get; }
23+
24+
public IEnumerable<string> Ids => [DiagnoserName];
1925

2026
public IEnumerable<IExporter> Exporters => [];
2127

2228
public IEnumerable<IAnalyser> Analysers => [];
2329

24-
public void DisplayResults(ILogger logger) => logger.WriteLine($"{nameof(MockInProcessDiagnoser)} results: [{string.Join(", ", Results.Values)}]");
30+
public void DisplayResults(ILogger logger) => logger.WriteLine($"{DiagnoserName} results: [{string.Join(", ", Results.Values)}]");
2531

26-
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => RunMode.NoOverhead;
32+
public RunMode GetRunMode(BenchmarkCase benchmarkCase) => DiagnoserRunMode;
2733

2834
public void Handle(HostSignal signal, DiagnoserActionParameters parameters) { }
2935

3036
public IEnumerable<Metric> ProcessResults(DiagnoserResults results) => [];
3137

3238
public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters) => [];
3339

34-
public (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
35-
=> (typeof(MockInProcessDiagnoserHandler), null);
40+
public abstract (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase);
41+
42+
public virtual IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
43+
{
44+
var (handlerType, serializedConfig) = GetSeparateProcessHandlerTypeAndSerializedConfig(benchmarkCase);
45+
if (handlerType == null)
46+
return null;
47+
var handler = (IInProcessDiagnoserHandler)Activator.CreateInstance(handlerType);
48+
handler.Initialize(serializedConfig);
49+
return handler;
50+
}
51+
52+
public void DeserializeResults(BenchmarkCase benchmarkCase, string results)
53+
{
54+
Results.Add(benchmarkCase, results);
55+
s_completedResults.Enqueue(this);
56+
}
57+
}
58+
59+
public abstract class BaseMockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
60+
{
61+
private string _result;
3662

37-
public IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
38-
=> new MockInProcessDiagnoserHandler();
63+
protected BaseMockInProcessDiagnoserHandler() { }
3964

40-
public void DeserializeResults(BenchmarkCase benchmarkCase, string results) => Results.Add(benchmarkCase, results);
65+
public void Initialize(string? serializedConfig)
66+
{
67+
_result = serializedConfig ?? string.Empty;
68+
}
69+
70+
public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
71+
72+
public string SerializeResults() => _result;
4173
}
4274

43-
public sealed class MockInProcessDiagnoserHandler : IInProcessDiagnoserHandler
75+
public sealed class MockInProcessDiagnoser : BaseMockInProcessDiagnoser
4476
{
45-
public void Initialize(string? serializedConfig) { }
77+
public override string DiagnoserName => nameof(MockInProcessDiagnoser);
78+
public override RunMode DiagnoserRunMode => RunMode.NoOverhead;
79+
public override string ExpectedResult => "MockResult";
4680

47-
public void Handle(BenchmarkSignal signal, InProcessDiagnoserActionArgs args) { }
81+
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
82+
=> (typeof(MockInProcessDiagnoserHandler), ExpectedResult);
83+
}
84+
85+
public sealed class MockInProcessDiagnoserHandler : BaseMockInProcessDiagnoserHandler
86+
{
87+
}
88+
89+
public sealed class MockInProcessDiagnoserExtraRun : BaseMockInProcessDiagnoser
90+
{
91+
public override string DiagnoserName => nameof(MockInProcessDiagnoserExtraRun);
92+
public override RunMode DiagnoserRunMode => RunMode.ExtraRun;
93+
public override string ExpectedResult => "ExtraRunResult";
94+
95+
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
96+
=> (typeof(MockInProcessDiagnoserExtraRunHandler), ExpectedResult);
97+
}
4898

49-
public string SerializeResults() => "MockResult";
99+
public sealed class MockInProcessDiagnoserExtraRunHandler : BaseMockInProcessDiagnoserHandler
100+
{
101+
}
102+
103+
public sealed class MockInProcessDiagnoserNone : BaseMockInProcessDiagnoser
104+
{
105+
public override string DiagnoserName => nameof(MockInProcessDiagnoserNone);
106+
public override RunMode DiagnoserRunMode => RunMode.None;
107+
public override string ExpectedResult => "NoneResult";
108+
109+
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
110+
=> default; // Returns default when RunMode is None
111+
112+
public override IInProcessDiagnoserHandler? GetSameProcessHandler(BenchmarkCase benchmarkCase)
113+
=> null; // Returns null when RunMode is None
114+
}
115+
116+
public sealed class MockInProcessDiagnoserNoneHandler : BaseMockInProcessDiagnoserHandler
117+
{
118+
}
119+
120+
public sealed class MockInProcessDiagnoserSeparateLogic : BaseMockInProcessDiagnoser
121+
{
122+
public override string DiagnoserName => nameof(MockInProcessDiagnoserSeparateLogic);
123+
public override RunMode DiagnoserRunMode => RunMode.SeparateLogic;
124+
public override string ExpectedResult => "SeparateLogicResult";
125+
126+
public override (Type? handlerType, string? serializedConfig) GetSeparateProcessHandlerTypeAndSerializedConfig(BenchmarkCase benchmarkCase)
127+
=> (typeof(MockInProcessDiagnoserSeparateLogicHandler), ExpectedResult);
128+
}
129+
130+
public sealed class MockInProcessDiagnoserSeparateLogicHandler : BaseMockInProcessDiagnoserHandler
131+
{
50132
}

‎tests/BenchmarkDotNet.IntegrationTests/InProcessDiagnoserTests.cs‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Linq;
4+
using System.Threading;
5+
using BenchmarkDotNet.Attributes;
6+
using BenchmarkDotNet.Columns;
7+
using BenchmarkDotNet.Configs;
8+
using BenchmarkDotNet.IntegrationTests.Diagnosers;
9+
using BenchmarkDotNet.Jobs;
10+
using BenchmarkDotNet.Tests.Loggers;
11+
using BenchmarkDotNet.Toolchains;
12+
using BenchmarkDotNet.Toolchains.InProcess.Emit;
13+
using BenchmarkDotNet.Toolchains.InProcess.NoEmit;
14+
using Xunit;
15+
using Xunit.Abstractions;
16+
using RunMode = BenchmarkDotNet.Diagnosers.RunMode;
17+
18+
namespace BenchmarkDotNet.IntegrationTests;
19+
20+
public class InProcessDiagnoserTests(ITestOutputHelper output) : BenchmarkTestExecutor(output)
21+
{
22+
// For test explorer since it doesn't handle interfaces well.
23+
public enum ToolchainType
24+
{
25+
Default,
26+
InProcessEmit,
27+
InProcessNoEmit,
28+
}
29+
30+
private static IEnumerable<RunMode[]> GetRunModeCombinations(int count)
31+
{
32+
var runModes = (RunMode[]) Enum.GetValues(typeof(RunMode));
33+
34+
if (count == 1)
35+
{
36+
foreach (var runMode in runModes)
37+
{
38+
yield return [runMode];
39+
}
40+
}
41+
else if (count == 3)
42+
{
43+
foreach (var runMode1 in runModes)
44+
{
45+
foreach (var runMode2 in runModes)
46+
{
47+
foreach (var runMode3 in runModes)
48+
{
49+
yield return [runMode1, runMode2, runMode3];
50+
}
51+
}
52+
}
53+
}
54+
}
55+
56+
public static IEnumerable<object[]> GetTestCombinations()
57+
{
58+
var toolchains = (ToolchainType[]) Enum.GetValues(typeof(ToolchainType));
59+
var counts = new[] { 1, 3 };
60+
61+
foreach (var toolchain in toolchains)
62+
{
63+
foreach (var count in counts)
64+
{
65+
foreach (var runModes in GetRunModeCombinations(count))
66+
{
67+
yield return [runModes, toolchain];
68+
}
69+
}
70+
}
71+
}
72+
73+
private static BaseMockInProcessDiagnoser CreateDiagnoser(RunMode runMode)
74+
=> runMode switch
75+
{
76+
RunMode.None => new MockInProcessDiagnoserNone(),
77+
RunMode.NoOverhead => new MockInProcessDiagnoser(),
78+
RunMode.ExtraRun => new MockInProcessDiagnoserExtraRun(),
79+
RunMode.SeparateLogic => new MockInProcessDiagnoserSeparateLogic(),
80+
_ => throw new ArgumentException($"Unsupported run mode: {runMode}")
81+
};
82+
83+
private ManualConfig CreateConfig(ToolchainType toolchain)
84+
{
85+
var job = toolchain switch
86+
{
87+
ToolchainType.InProcessEmit => Job.Dry.WithToolchain(InProcessEmitToolchain.Instance),
88+
ToolchainType.InProcessNoEmit => Job.Dry.WithToolchain(InProcessNoEmitToolchain.Instance),
89+
_ => Job.Dry
90+
};
91+
92+
return new ManualConfig()
93+
.AddLogger(new OutputLogger(Output))
94+
.AddColumnProvider(DefaultColumnProviders.Instance)
95+
.AddJob(job);
96+
}
97+
98+
[Theory]
99+
[MemberData(nameof(GetTestCombinations))]
100+
public void MultipleInProcessDiagnosersWork(RunMode[] runModes, ToolchainType toolchain)
101+
{
102+
var diagnosers = runModes.Select(CreateDiagnoser).ToArray();
103+
var config = CreateConfig(toolchain);
104+
105+
foreach (var diagnoser in diagnosers)
106+
{
107+
config = config.AddDiagnoser(diagnoser);
108+
}
109+
110+
var summary = CanExecute<SimpleBenchmark>(config);
111+
112+
foreach (var diagnoser in diagnosers)
113+
{
114+
if (diagnoser.DiagnoserRunMode == RunMode.None)
115+
{
116+
Assert.Empty(diagnoser.Results);
117+
}
118+
else
119+
{
120+
Assert.NotEmpty(diagnoser.Results);
121+
Assert.Equal(summary.BenchmarksCases.Length, diagnoser.Results.Count);
122+
Assert.All(diagnoser.Results.Values, result => Assert.Equal(diagnoser.ExpectedResult, result));
123+
}
124+
}
125+
Assert.Equal(
126+
BaseMockInProcessDiagnoser.s_completedResults,
127+
diagnosers
128+
.Where(d => d.DiagnoserRunMode != RunMode.None)
129+
.OrderBy(d => d.DiagnoserRunMode switch
130+
{
131+
RunMode.NoOverhead => 0,
132+
RunMode.ExtraRun => 1,
133+
RunMode.SeparateLogic => 2,
134+
_ => 3
135+
})
136+
);
137+
BaseMockInProcessDiagnoser.s_completedResults.Clear();
138+
}
139+
140+
public class SimpleBenchmark
141+
{
142+
private int counter;
143+
144+
[Benchmark]
145+
public void BenchmarkMethod()
146+
{
147+
Interlocked.Increment(ref counter);
148+
}
149+
}
150+
}

‎tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -131,19 +131,6 @@ public void InProcessBenchmarkEmitsSameIL(Type benchmarkType)
131131
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
132132
}
133133

134-
[Fact]
135-
public void InProcessEmitSupportsInProcessDiagnosers()
136-
{
137-
var logger = new OutputLogger(Output);
138-
var diagnoser = new MockInProcessDiagnoser();
139-
var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser);
140-
141-
var summary = CanExecute<BenchmarkAllCases>(config);
142-
143-
var expected = Enumerable.Repeat("MockResult", summary.BenchmarksCases.Length);
144-
Assert.Equal(expected, diagnoser.Results.Values);
145-
}
146-
147134
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
148135
public class BenchmarkAllCases
149136
{

‎tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -218,19 +218,6 @@ public void InProcessBenchmarkAllCasesSupported()
218218
}
219219
}
220220

221-
[Fact]
222-
public void InProcessNoEmitSupportsInProcessDiagnosers()
223-
{
224-
var logger = new OutputLogger(Output);
225-
var diagnoser = new MockInProcessDiagnoser();
226-
var config = CreateInProcessConfig(logger).AddDiagnoser(diagnoser);
227-
228-
var summary = CanExecute<BenchmarkAllCases>(config);
229-
230-
var expected = Enumerable.Repeat("MockResult", summary.BenchmarksCases.Length);
231-
Assert.Equal(expected, diagnoser.Results.Values);
232-
}
233-
234221
[UsedImplicitly(ImplicitUseTargetFlags.WithMembers)]
235222
public class BenchmarkAllCases
236223
{

‎tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,9 @@ public void NativeAotSupportsInProcessDiagnosers()
7979
throw;
8080
}
8181

82-
Assert.Equal(["MockResult"], diagnoser.Results.Values);
82+
Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values);
83+
Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults);
84+
BaseMockInProcessDiagnoser.s_completedResults.Clear();
8385
}
8486

8587
private static bool GetShouldRunTest()

‎tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@ public void WasmSupportsInProcessDiagnosers()
5656

5757
CanExecute<WasmBenchmark>(config);
5858

59-
Assert.Equal(["MockResult"], diagnoser.Results.Values);
59+
Assert.Equal([diagnoser.ExpectedResult], diagnoser.Results.Values);
60+
Assert.Equal([diagnoser], BaseMockInProcessDiagnoser.s_completedResults);
61+
BaseMockInProcessDiagnoser.s_completedResults.Clear();
6062
}
6163

6264
public class WasmBenchmark

0 commit comments

Comments
 (0)