Skip to content

Commit

Permalink
Benchmarks: customization and formatting
Browse files Browse the repository at this point in the history 
More cleanup to do (removing Type column next), but getting to some very
usable output now.
  • Loading branch information
Nick Craver committed May 11, 2017
1 parent d72ae17 commit d3ab338
Show file tree
Hide file tree
Showing 14 changed files with 116 additions and 54 deletions.
6 changes: 3 additions & 3 deletions Dapper.Tests.Performance/Benchmarks.Belgrade.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,12 @@ public void Setup()
_mapper = new QueryMapper(ConnectionString);
}

[Benchmark(Description = "Belgrade: ExecuteReader", OperationsPerInvoke = Iterations)]
public Task ExecuteReader()
[Benchmark(Description = "ExecuteReader", OperationsPerInvoke = Iterations)]
public async Task ExecuteReader()
{
Step();
// TODO: How do you get a Post out of this thing?
return _mapper.ExecuteReader("SELECT TOP 1 * FROM Posts WHERE Id = " + i,
await _mapper.ExecuteReader("SELECT TOP 1 * FROM Posts WHERE Id = " + i,
reader =>
{
var post = new Post();
Expand Down
39 changes: 18 additions & 21 deletions Dapper.Tests.Performance/Benchmarks.Dapper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,50 +12,47 @@ public void Setup()
BaseSetup();
}

[Benchmark(Description = "Dapper: Query<T> (buffered)", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Query<T> (buffered)", OperationsPerInvoke = Iterations)]
public Post QueryBuffered()
{
Step();
return _connection.Query<Post>("select * from Posts where Id = @Id", new { Id = i }, buffered: true).First();
}

[Benchmark(Description = "Dapper: Query<T> (unbuffered)", OperationsPerInvoke = Iterations)]
public Post QueryUnbuffered()
[Benchmark(Description = "Query<dyanmic> (buffered)", OperationsPerInvoke = Iterations)]
public dynamic QueryBufferedDynamic()
{
Step();
return _connection.Query<Post>("select * from Posts where Id = @Id", new { Id = i }, buffered: false).First();
return _connection.Query("select * from Posts where Id = @Id", new { Id = i }, buffered: true).First();
}

[Benchmark(Description = "Dapper: QueryFirstOrDefault<T>", OperationsPerInvoke = Iterations)]
public Post QueryFirstOrDefault()
[Benchmark(Description = "Query<T> (unbuffered)", OperationsPerInvoke = Iterations)]
public Post QueryUnbuffered()
{
Step();
return _connection.QueryFirstOrDefault<Post>("select * from Posts where Id = @Id", new { Id = i });
return _connection.Query<Post>("select * from Posts where Id = @Id", new { Id = i }, buffered: false).First();
}

[Benchmark(Description = "Dapper: Query<dyanmic> (buffered)", OperationsPerInvoke = Iterations)]
public object QueryBufferedDynamic()
[Benchmark(Description = "Query<dyanmic> (unbuffered)", OperationsPerInvoke = Iterations)]
public dynamic QueryUnbufferedDynamic()
{
Step();
return _connection.Query("select * from Posts where Id = @Id", new { Id = i }, buffered: true).First();
return _connection.Query("select * from Posts where Id = @Id", new { Id = i }, buffered: false).First();
}

[Benchmark(Description = "Dapper: Query<dyanmic> (unbuffered)", OperationsPerInvoke = Iterations)]
public object QueryUnbufferedDynamic()
[Benchmark(Description = "QueryFirstOrDefault<T>", OperationsPerInvoke = Iterations)]
public Post QueryFirstOrDefault()
{
Step();
return _connection.Query("select * from Posts where Id = @Id", new { Id = i }, buffered: false).First();
return _connection.QueryFirstOrDefault<Post>("select * from Posts where Id = @Id", new { Id = i });
}

[Benchmark(Description = "Dapper: QueryFirstOrDefault<dyanmic>", OperationsPerInvoke = Iterations)]
public object QueryFirstOrDefaultDynamic()
[Benchmark(Description = "QueryFirstOrDefault<dyanmic>", OperationsPerInvoke = Iterations)]
public dynamic QueryFirstOrDefaultDynamic()
{
Step();
return _connection.QueryFirstOrDefault("select * from Posts where Id = @Id", new { Id = i }).First();
}

[Benchmark(Description = "Dapper: Contrib Get<T>", OperationsPerInvoke = Iterations)]
public object ContribGet()
[Benchmark(Description = "Contrib Get<T>", OperationsPerInvoke = Iterations)]
public Post ContribGet()
{
Step();
return _connection.Get<Post>(i);
Expand Down
8 changes: 4 additions & 4 deletions Dapper.Tests.Performance/Benchmarks.EntityFramework.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

namespace Dapper.Tests.Performance
{
public class EntityFrameworkBenchmarks : BenchmarkBase
public class EF6Benchmarks : BenchmarkBase
{
private EntityFramework.EFContext Context;
private static readonly Func<DataClassesDataContext, int, Linq2Sql.Post> compiledQuery =
Expand All @@ -19,21 +19,21 @@ public void Setup()
Context = new EntityFramework.EFContext(_connection);
}

[Benchmark(Description = "EF6: Normal", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Normal", OperationsPerInvoke = Iterations)]
public Post Normal()
{
Step();
return Context.Posts.First(p => p.Id == i);
}

[Benchmark(Description = "EF6: SqlQuery", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "SqlQuery", OperationsPerInvoke = Iterations)]
public Post SqlQuery()
{
Step();
return Context.Database.SqlQuery<Post>("select * from Posts where Id = {0}", i).First();
}

[Benchmark(Description = "EF6: No Tracking", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "No Tracking", OperationsPerInvoke = Iterations)]
public Post NoTracking()
{
Step();
Expand Down
8 changes: 4 additions & 4 deletions Dapper.Tests.Performance/Benchmarks.HandCoded.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ public void Setup()
#endif
}

[Benchmark(Description = "HandCoded: SqlCommand", OperationsPerInvoke = Iterations)]
public dynamic SqlCommand()
[Benchmark(Description = "SqlCommand", OperationsPerInvoke = Iterations, Baseline = true)]
public Post SqlCommand()
{
Step();
_idParam.Value = i;
Expand All @@ -75,8 +75,8 @@ public dynamic SqlCommand()
}
}

[Benchmark(Description = "HandCoded: DataTable", OperationsPerInvoke = Iterations)]
public dynamic DataTable()
[Benchmark(Description = "DataTable", OperationsPerInvoke = Iterations)]
public dynamic DataTableDynamic()
{
Step();
_idParam.Value = i;
Expand Down
6 changes: 3 additions & 3 deletions Dapper.Tests.Performance/Benchmarks.Linq2Sql.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,21 @@ public void Setup()
Linq2SqlContext = new DataClassesDataContext(_connection);
}

[Benchmark(Description = "Linq2Sql: Normal", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Normal", OperationsPerInvoke = Iterations)]
public Linq2Sql.Post Normal()
{
Step();
return Linq2SqlContext.Posts.First(p => p.Id == i);
}

[Benchmark(Description = "Linq2Sql: Compiled", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Compiled", OperationsPerInvoke = Iterations)]
public Linq2Sql.Post Compiled()
{
Step();
return compiledQuery(Linq2SqlContext, i);
}

[Benchmark(Description = "Linq2Sql: ExecuteQuery", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "ExecuteQuery", OperationsPerInvoke = Iterations)]
public Post ExecuteQuery()
{
Step();
Expand Down
4 changes: 2 additions & 2 deletions Dapper.Tests.Performance/Benchmarks.Massive.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ public void Setup()
_model = new DynamicModel(ConnectionString);
}

[Benchmark(Description = "Massive: Query (dynamic)", OperationsPerInvoke = Iterations)]
public dynamic Query()
[Benchmark(Description = "Query (dynamic)", OperationsPerInvoke = Iterations)]
public dynamic QueryDynamic()
{
Step();
return _model.Query("select * from Posts where Id = @0", _connection, i).First();
Expand Down
10 changes: 5 additions & 5 deletions Dapper.Tests.Performance/Benchmarks.NHibernate.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public void Setup()
_get = NHibernateHelper.OpenSession();
}

[Benchmark(Description = "NHibernate: SQL", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "SQL", OperationsPerInvoke = Iterations)]
public Post SQL()
{
Step();
Expand All @@ -34,7 +34,7 @@ public Post SQL()
.List<Post>()[0];
}

[Benchmark(Description = "NHibernate: HQL", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "HQL", OperationsPerInvoke = Iterations)]
public Post HQL()
{
Step();
Expand All @@ -43,7 +43,7 @@ public Post HQL()
.List<Post>()[0];
}

[Benchmark(Description = "NHibernate: Criteria", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Criteria", OperationsPerInvoke = Iterations)]
public Post Criteria()
{
Step();
Expand All @@ -52,14 +52,14 @@ public Post Criteria()
.List<Post>()[0];
}

[Benchmark(Description = "NHibernate: LINQ", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "LINQ", OperationsPerInvoke = Iterations)]
public Post LINQ()
{
Step();
return _linq.Query<Post>().First(p => p.Id == i);
}

[Benchmark(Description = "NHibernate: Get<T>", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Get<T>", OperationsPerInvoke = Iterations)]
public Post Get()
{
Step();
Expand Down
8 changes: 4 additions & 4 deletions Dapper.Tests.Performance/Benchmarks.PetaPoco.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,15 @@ public void Setup()
_dbFast.ForceDateTimesToUtc = false;
}

[Benchmark(Description = "PetaPoco: Fetch<Post>", OperationsPerInvoke = Iterations)]
public dynamic Fetch()
[Benchmark(Description = "Fetch<Post>", OperationsPerInvoke = Iterations)]
public Post Fetch()
{
Step();
return _db.Fetch<Post>("SELECT * from Posts where Id=@0", i).First();
}

[Benchmark(Description = "PetaPoco: Fetch<Post> (Fast)", OperationsPerInvoke = Iterations)]
public dynamic FetchFast()
[Benchmark(Description = "Fetch<Post> (Fast)", OperationsPerInvoke = Iterations)]
public Post FetchFast()
{
Step();
return _dbFast.Fetch<Post>("SELECT * from Posts where Id=@0", i).First();
Expand Down
2 changes: 1 addition & 1 deletion Dapper.Tests.Performance/Benchmarks.ServiceStack.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ public void Setup()
_db = dbFactory.Open();
}

[Benchmark(Description = "ServiceStack.OrmLite: SingleById", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "SingleById", OperationsPerInvoke = Iterations)]
public Post Query()
{
Step();
Expand Down
4 changes: 2 additions & 2 deletions Dapper.Tests.Performance/Benchmarks.Soma.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ public void Setup()
_sdb = Simple.Data.Database.OpenConnection(ConnectionString);
}

[Benchmark(Description = "Soma: FindById", OperationsPerInvoke = Iterations)]
public dynamic Query()
[Benchmark(Description = "FindById", OperationsPerInvoke = Iterations)]
public dynamic QueryDynamic()
{
Step();
return _sdb.Posts.FindById(i).FirstOrDefault();
Expand Down
8 changes: 4 additions & 4 deletions Dapper.Tests.Performance/Benchmarks.Susanoo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ public void Setup()
_db = new DatabaseManager(_connection);
}

[Benchmark(Description = "Susanoo: Mapping Cache", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Mapping Cache", OperationsPerInvoke = Iterations)]
public Post MappingCache()
{
Step();
Expand All @@ -35,7 +35,7 @@ public Post MappingCache()
.Execute(_db, new { Id = i }).First();
}

[Benchmark(Description = "Susanoo: Mapping Cache (dynamic)", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Mapping Cache (dynamic)", OperationsPerInvoke = Iterations)]
public dynamic MappingCacheDynamic()
{
Step();
Expand All @@ -45,14 +45,14 @@ public dynamic MappingCacheDynamic()
.Execute(_db, new { Id = i }).First();
}

[Benchmark(Description = "Susanoo: Mapping Static", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Mapping Static", OperationsPerInvoke = Iterations)]
public Post MappingStatic()
{
Step();
return _cmd.Execute(_db, new { Id = i }).First();
}

[Benchmark(Description = "Susanoo: Mapping Static (dynamic)", OperationsPerInvoke = Iterations)]
[Benchmark(Description = "Mapping Static (dynamic)", OperationsPerInvoke = Iterations)]
public dynamic MappingStaticDynamic()
{
Step();
Expand Down
17 changes: 16 additions & 1 deletion Dapper.Tests.Performance/Benchmarks.cs
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,27 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Attributes.Columns;
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Horology;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Order;
using Dapper.Tests.Performance.Helpers;
using System;
using System.Configuration;
using System.Data.SqlClient;

namespace Dapper.Tests.Performance
{
[OrderProvider(SummaryOrderPolicy.FastestToSlowest)]
[RankColumn]
[Config(typeof(Config))]
public abstract class BenchmarkBase
{
public const int Iterations = 50;

This comment has been minimized.

Sign in to view

Copy link
@DixonDs

DixonDs May 13, 2017

I might be missing something, but it is used in OperationsPerInvoke only while most (all?) benchmarks have a single operation inside. Is it intentional or just a mistake?

This comment has been minimized.

Sign in to view

Copy link
@NickCraver

NickCraver May 13, 2017

Member

@DixonDs that's an instruction for BenchmarkDotNet on how many times to run it. That tells it to call the setup method once and the benchmark method n times. Each time it's stepping to a new i and selecting a different post as a result (to make sure we aren't hitting cache which some providers have). Running the benchmark only once would be testing initial startup time only, rather than a mix.

Some providers have more startup time, but each run after is much faster. Some do the world every time, and some are elsewhere on the spectrum. 50 iterations seems like a reasonable number for actual real-world use cases. But, I put it up here as a constant just so that it was easy to adjust across the board to say 1 and test startup cost, or 10000 to test after-first average performance.

Unfortunately I can't easily make it a number you can set at the command line, due to how attributes work. But we may be able to change the way the job inits the config and have it (optionally) passed in that way to set it to something other than 50.

At that point though, we'll need to change to a command-line parser to stay sane. Which I'm not against, just hadn't gotten there yet. I was just trying to get all the framework up here that makes it easy to add providers, see allocations, etc. :)

This comment has been minimized.

Sign in to view

Copy link
@DixonDs

DixonDs May 14, 2017

@NickCraver

that's an instruction for BenchmarkDotNet on how many times to run it. That tells it to call the setup method once and the benchmark method n times.

You sure about that? In my understanding, it tells BenchmarkDotNet how many operations you did in the benchmarked method by yourself. Check this line: https://github.com/dotnet/BenchmarkDotNet/blob/0898c3fd2c371dc6d24a8e2933014ca63e8051d8/src/BenchmarkDotNet.Core/Engines/Engine.cs#L143

This comment has been minimized.

Sign in to view

Copy link
@NickCraver

NickCraver May 14, 2017

Member

@DixonDs Ahh, yes I've totally misread this on the BenchmarkDotNet side - will convert all these to for loops, though I wish there was a bit cleaner way to do it everywhere. Thanks for pointing this out, now I see where I fucked up :) I'll push a new commit in a bit that fixes these.

This comment has been minimized.

Sign in to view

Copy link
@NickCraver

NickCraver May 14, 2017

Member

@DixonDs not perfect, but much better setup pushed in fb44338 - I think we can do better analysis yet, but this is a much faster run with the correct iteration counts. And we could do the command-line setting from that angle now.
protected static readonly Random _rand = new Random();
protected SqlConnection _connection;
public static string ConnectionString { get; } = ConfigurationManager.ConnectionStrings["Main"].ConnectionString;

protected int i;

protected void BaseSetup()
Expand All @@ -36,6 +43,14 @@ public class Config : ManualConfig
public Config()
{
Add(new MemoryDiagnoser());
Add(new ORMColum());
Add(new ReturnColum());
Add(Job.Default
.WithLaunchCount(1)
.WithIterationTime(new TimeInterval(500, TimeUnit.Millisecond))
.WithWarmupCount(3)
.WithTargetCount(3)
);
}
}
}
25 changes: 25 additions & 0 deletions Dapper.Tests.Performance/Helpers/ORMColum.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;

namespace Dapper.Tests.Performance.Helpers
{
public class ORMColum : IColumn
{
public string Id => nameof(ORMColum);
public string ColumnName { get; } = "ORM";
public string Legend => "The object relational mapper being tested";

public bool IsDefault(Summary summary, Benchmark benchmark) => false;
public string GetValue(Summary summary, Benchmark benchmark) => benchmark.Target.Method.DeclaringType.Name.Replace("Benchmarks", string.Empty);
public string GetValue(Summary summary, Benchmark benchmark, ISummaryStyle style) => benchmark.Target.Method.DeclaringType.Name.Replace("Benchmarks", string.Empty);

public bool IsAvailable(Summary summary) => true;
public bool AlwaysShow => true;
public ColumnCategory Category => ColumnCategory.Job;
public int PriorityInCategory => -10;
public bool IsNumeric => false;
public UnitType UnitType => UnitType.Dimensionless;
public override string ToString() => ColumnName;
}
}
25 changes: 25 additions & 0 deletions Dapper.Tests.Performance/Helpers/ReturnColum.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
using BenchmarkDotNet.Columns;
using BenchmarkDotNet.Reports;
using BenchmarkDotNet.Running;

namespace Dapper.Tests.Performance.Helpers
{
public class ReturnColum : IColumn
{
public string Id => nameof(ReturnColum);
public string ColumnName { get; } = "Return";
public string Legend => "The return type of the method";

public bool IsDefault(Summary summary, Benchmark benchmark) => false;
public string GetValue(Summary summary, Benchmark benchmark) => benchmark.Target.Method.ReturnType.Name;
public string GetValue(Summary summary, Benchmark benchmark, ISummaryStyle style) => benchmark.Target.Method.ReturnType.Name;

public bool IsAvailable(Summary summary) => true;
public bool AlwaysShow => true;
public ColumnCategory Category => ColumnCategory.Job;
public int PriorityInCategory => 1;
public bool IsNumeric => false;
public UnitType UnitType => UnitType.Dimensionless;
public override string ToString() => ColumnName;
}
}

0 comments on commit d3ab338

Please sign in to comment.