Skip to content

Add grouping by type #2163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
Open

Add grouping by type #2163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
0558dc2
mock factory
YegorStepanov Oct 18, 2022
2025e87
Add new tests
YegorStepanov Oct 18, 2022
b1d588e
Add approved files
YegorStepanov Oct 19, 2022
86bb98d
Add ByType rule
YegorStepanov Oct 18, 2022
bf9a591
Update existing approved files
YegorStepanov Oct 20, 2022
16a74d9
Update new approved files
YegorStepanov Oct 20, 2022
22e8883
Add new approved files (ByType)
YegorStepanov Oct 20, 2022
6571601
most conservative mode
YegorStepanov Oct 20, 2022
5eb47d4
Update new approved files
YegorStepanov Oct 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/BenchmarkDotNet/Configs/BenchmarkLogicalGroupRule.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,6 @@
{
public enum BenchmarkLogicalGroupRule
{
ByMethod, ByJob, ByParams, ByCategory
ByMethod, ByJob, ByParams, ByCategory, ByType
}
}
35 changes: 26 additions & 9 deletions src/BenchmarkDotNet/Order/DefaultOrderer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
using System.Diagnostics.CodeAnalysis;
using System.Linq;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Extensions;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Parameters;
using BenchmarkDotNet.Reports;
Expand All @@ -20,7 +21,8 @@ public class DefaultOrderer : IOrderer
private readonly IComparer<string[]> categoryComparer = CategoryComparer.Instance;
private readonly IComparer<ParameterInstances> paramsComparer = ParameterComparer.Instance;
private readonly IComparer<Job> jobComparer = JobComparer.Instance;
private readonly IComparer<Descriptor> targetComparer;
private readonly IComparer<Descriptor> typeComparer = TypeComparer.Instance;
private readonly IComparer<Descriptor> methodComparer;

public SummaryOrderPolicy SummaryOrderPolicy { get; }
public MethodOrderPolicy MethodOrderPolicy { get; }
Expand All @@ -31,15 +33,15 @@ public DefaultOrderer(
{
SummaryOrderPolicy = summaryOrderPolicy;
MethodOrderPolicy = methodOrderPolicy;
targetComparer = new DescriptorComparer(methodOrderPolicy);
methodComparer = new MethodComparer(methodOrderPolicy);
}

[PublicAPI]
public virtual IEnumerable<BenchmarkCase> GetExecutionOrder(
ImmutableArray<BenchmarkCase> benchmarkCases,
IEnumerable<BenchmarkLogicalGroupRule> order = null)
{
var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order);
var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, typeComparer, methodComparer, order);
var list = benchmarkCases.ToList();
list.Sort(benchmarkComparer);
return list;
Expand Down Expand Up @@ -87,8 +89,15 @@ public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCase
{
var explicitRules = benchmarkCase.Config.GetLogicalGroupRules().ToList();
var implicitRules = new List<BenchmarkLogicalGroupRule>();

bool hasMultipleTypes = allBenchmarksCases.DistinctBy(b => b.Descriptor.Type).Count() > 1;
bool hasJobBaselines = allBenchmarksCases.Any(b => b.Job.Meta.Baseline);
bool hasDescriptorBaselines = allBenchmarksCases.Any(b => b.Descriptor.Baseline);

if (hasMultipleTypes && !hasJobBaselines && !hasDescriptorBaselines)
{
implicitRules.Add(BenchmarkLogicalGroupRule.ByType);
}
if (hasJobBaselines)
{
implicitRules.Add(BenchmarkLogicalGroupRule.ByParams);
Expand All @@ -114,8 +123,11 @@ public string GetLogicalGroupKey(ImmutableArray<BenchmarkCase> allBenchmarksCase
{
switch (rule)
{
case BenchmarkLogicalGroupRule.ByType:
keys.Add(benchmarkCase.Descriptor.TypeInfo);
break;
case BenchmarkLogicalGroupRule.ByMethod:
keys.Add(benchmarkCase.Descriptor.DisplayInfo);
keys.Add(benchmarkCase.Descriptor.WorkloadMethodDisplayInfo);
break;
case BenchmarkLogicalGroupRule.ByJob:
keys.Add(benchmarkCase.Job.DisplayInfo);
Expand All @@ -139,7 +151,7 @@ public virtual IEnumerable<IGrouping<string, BenchmarkCase>> GetLogicalGroupOrde
IEnumerable<IGrouping<string, BenchmarkCase>> logicalGroups,
IEnumerable<BenchmarkLogicalGroupRule> order = null)
{
var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, targetComparer, order);
var benchmarkComparer = new BenchmarkComparer(categoryComparer, paramsComparer, jobComparer, typeComparer, methodComparer, order);
var logicalGroupComparer = new LogicalGroupComparer(benchmarkComparer);
var list = logicalGroups.ToList();
list.Sort(logicalGroupComparer);
Expand All @@ -152,6 +164,7 @@ private class BenchmarkComparer : IComparer<BenchmarkCase>
{
private static readonly BenchmarkLogicalGroupRule[] DefaultOrder =
{
BenchmarkLogicalGroupRule.ByType,
BenchmarkLogicalGroupRule.ByCategory,
BenchmarkLogicalGroupRule.ByParams,
BenchmarkLogicalGroupRule.ByJob,
Expand All @@ -161,18 +174,21 @@ private class BenchmarkComparer : IComparer<BenchmarkCase>
private readonly IComparer<string[]> categoryComparer;
private readonly IComparer<ParameterInstances> paramsComparer;
private readonly IComparer<Job> jobComparer;
private readonly IComparer<Descriptor> targetComparer;
private readonly IComparer<Descriptor> typeComparer;
private readonly IComparer<Descriptor> methodComparer;
private readonly List<BenchmarkLogicalGroupRule> order;

public BenchmarkComparer(
IComparer<string[]> categoryComparer,
IComparer<ParameterInstances> paramsComparer,
IComparer<Job> jobComparer,
IComparer<Descriptor> targetComparer,
IComparer<Descriptor> typeComparer,
IComparer<Descriptor> methodComparer,
IEnumerable<BenchmarkLogicalGroupRule> order)
{
this.categoryComparer = categoryComparer;
this.targetComparer = targetComparer;
this.typeComparer = typeComparer;
this.methodComparer = methodComparer;
this.jobComparer = jobComparer;
this.paramsComparer = paramsComparer;

Expand All @@ -192,7 +208,8 @@ public int Compare(BenchmarkCase x, BenchmarkCase y)
{
int compare = rule switch
{
BenchmarkLogicalGroupRule.ByMethod => targetComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0,
BenchmarkLogicalGroupRule.ByType => typeComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0,
BenchmarkLogicalGroupRule.ByMethod => methodComparer?.Compare(x.Descriptor, y.Descriptor) ?? 0,
BenchmarkLogicalGroupRule.ByJob => jobComparer?.Compare(x.Job, y.Job) ?? 0,
BenchmarkLogicalGroupRule.ByParams => paramsComparer?.Compare(x.Parameters, y.Parameters) ?? 0,
BenchmarkLogicalGroupRule.ByCategory => categoryComparer?.Compare(x.Descriptor.Categories, y.Descriptor.Categories) ?? 0,
Expand Down
10 changes: 5 additions & 5 deletions src/BenchmarkDotNet/Running/DescriptorComparer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,14 @@

namespace BenchmarkDotNet.Running
{
internal class DescriptorComparer : IComparer<Descriptor>
internal class MethodComparer : IComparer<Descriptor>
{
[PublicAPI] public static readonly IComparer<Descriptor> Alphabetical = new DescriptorComparer(MethodOrderPolicy.Alphabetical);
[PublicAPI] public static readonly IComparer<Descriptor> Declared = new DescriptorComparer(MethodOrderPolicy.Declared);
[PublicAPI] public static readonly IComparer<Descriptor> Alphabetical = new MethodComparer(MethodOrderPolicy.Alphabetical);
[PublicAPI] public static readonly IComparer<Descriptor> Declared = new MethodComparer(MethodOrderPolicy.Declared);

private readonly MethodOrderPolicy methodOrderPolicy;

public DescriptorComparer(MethodOrderPolicy methodOrderPolicy)
public MethodComparer(MethodOrderPolicy methodOrderPolicy)
{
this.methodOrderPolicy = methodOrderPolicy;
}
Expand All @@ -25,7 +25,7 @@ public int Compare(Descriptor x, Descriptor y)
switch (methodOrderPolicy)
{
case MethodOrderPolicy.Alphabetical:
return string.CompareOrdinal(x.DisplayInfo, y.DisplayInfo);
return string.CompareOrdinal(x.WorkloadMethodDisplayInfo, y.WorkloadMethodDisplayInfo);
case MethodOrderPolicy.Declared:
return x.MethodIndex - y.MethodIndex;
default:
Expand Down
17 changes: 17 additions & 0 deletions src/BenchmarkDotNet/Running/TypeComparer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
using System.Collections.Generic;

namespace BenchmarkDotNet.Running
{
internal class TypeComparer : IComparer<Descriptor>
{
public static readonly IComparer<Descriptor> Instance = new TypeComparer();

public int Compare(Descriptor x, Descriptor y)
{
if (x == null && y == null) return 0;
if (x != null && y == null) return 1;
if (x == null) return -1;
return string.CompareOrdinal(x.TypeInfo, y.TypeInfo);
}
}
}
18 changes: 9 additions & 9 deletions ...iles/MarkdownExporterApprovalTests.GroupExporterTest.Invalid_TwoJobBaselines.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,14 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Job2 : extra output line


Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |---------:|--------:|--------:|------:|--------:|-----:|---------------------------- |--------- |
Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Foo | Yes |
Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 2 | Invalid_TwoJobBaselines.Foo | Yes |
| | | | | | | | | |
Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Invalid_TwoJobBaselines.Bar | Yes |
Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Invalid_TwoJobBaselines.Bar | Yes |
Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- |
Foo | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Foo | Yes |
Foo | Job2 | 302.0 ns | 6.09 ns | 1.58 ns | 2.96 | 0.03 | 2 | Foo | Yes |
| | | | | | | | | |
Bar | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Bar | Yes |
Bar | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Bar | Yes |

Errors: 2
* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Foo in class Invalid_TwoJobBaselines has 2
* Only 1 job in a group can have "Baseline = true" applied to it, group Invalid_TwoJobBaselines.Bar in class Invalid_TwoJobBaselines has 2
* Only 1 job in a group can have "Baseline = true" applied to it, group Foo in class Invalid_TwoJobBaselines has 2
* Only 1 job in a group can have "Baseline = true" applied to it, group Bar in class Invalid_TwoJobBaselines has 2
20 changes: 10 additions & 10 deletions ...iles/MarkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsJobs.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,15 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Job2 : extra output line


Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |---------:|--------:|--------:|------:|--------:|-----:|----------------------------- |--------- |
Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Base | Yes |
Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | JobBaseline_MethodsJobs.Base | No |
| | | | | | | | | |
Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Foo | Yes |
Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | JobBaseline_MethodsJobs.Foo | No |
| | | | | | | | | |
Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | JobBaseline_MethodsJobs.Bar | Yes |
Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | JobBaseline_MethodsJobs.Bar | No |
Method | Job | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |---------:|--------:|--------:|------:|--------:|-----:|------------- |--------- |
Base | Job1 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Base | Yes |
Base | Job2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | Base | No |
| | | | | | | | | |
Foo | Job1 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Foo | Yes |
Foo | Job2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | Foo | No |
| | | | | | | | | |
Bar | Job1 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | Bar | Yes |
Bar | Job2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | Bar | No |

Errors: 0
38 changes: 19 additions & 19 deletions ...arkdownExporterApprovalTests.GroupExporterTest.JobBaseline_MethodsParamsJobs.approved.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,24 @@ Frequency=2531248 Hz, Resolution=395.0620 ns, Timer=TSC
Job2 : extra output line


Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------------------------------------- |--------- |
Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | Yes | ^
Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Base | No |
| | | | | | | | | | |
Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | Yes |
Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Foo | No |
| | | | | | | | | | |
Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | Yes |
Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-JobBaseline_MethodsParamsJobs.Bar | No |
| | | | | | | | | | |
Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | Yes | ^
Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Base | No |
| | | | | | | | | | |
Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | Yes |
Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Foo | No |
| | | | | | | | | | |
Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | Yes |
Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-JobBaseline_MethodsParamsJobs.Bar | No |
Method | Job | Param | Mean | Error | StdDev | Ratio | RatioSD | Rank | LogicalGroup | Baseline |
------- |----- |------ |-----------:|--------:|--------:|------:|--------:|-----:|---------------- |--------- |
Base | Job1 | 2 | 102.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Base | Yes | ^
Base | Job2 | 2 | 402.0 ns | 6.09 ns | 1.58 ns | 3.94 | 0.05 | 2 | [Param=2]-Base | No |
| | | | | | | | | | |
Foo | Job1 | 2 | 202.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Foo | Yes |
Foo | Job2 | 2 | 502.0 ns | 6.09 ns | 1.58 ns | 2.49 | 0.01 | 2 | [Param=2]-Foo | No |
| | | | | | | | | | |
Bar | Job1 | 2 | 302.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=2]-Bar | Yes |
Bar | Job2 | 2 | 602.0 ns | 6.09 ns | 1.58 ns | 1.99 | 0.01 | 2 | [Param=2]-Bar | No |
| | | | | | | | | | |
Base | Job1 | 10 | 702.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Base | Yes | ^
Base | Job2 | 10 | 1,002.0 ns | 6.09 ns | 1.58 ns | 1.43 | 0.00 | 2 | [Param=10]-Base | No |
| | | | | | | | | | |
Foo | Job1 | 10 | 802.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Foo | Yes |
Foo | Job2 | 10 | 1,102.0 ns | 6.09 ns | 1.58 ns | 1.37 | 0.00 | 2 | [Param=10]-Foo | No |
| | | | | | | | | | |
Bar | Job1 | 10 | 902.0 ns | 6.09 ns | 1.58 ns | 1.00 | 0.00 | 1 | [Param=10]-Bar | Yes |
Bar | Job2 | 10 | 1,202.0 ns | 6.09 ns | 1.58 ns | 1.33 | 0.00 | 2 | [Param=10]-Bar | No |

Errors: 0
Loading