Tidy up performance tests (#3230)

* User Story 34145: Tidy up Bulk Copy unmatched column name work

- Tidied up the existing Performance benchmarks.
- Added environment variables to specify config files.
- Removed Newtonsoft.Json as a dependency.
- Simplified csproj a bit.
- Improved benchmark running instructions.

Author: paulmedynski

BUILDGUIDE.md

@@ -339,14 +339,118 @@ dotnet test <test_properties...> --collect:"XPlat Code Coverage"
 
 ## Run Performance Tests
 
-### Running Performance test project directly
+The performance tests live here:
+`src\Microsoft.Data.SqlClient\tests\PerformanceTests\`
 
-Project location from Root: `src\Microsoft.Data.SqlClient\tests\PerformanceTests\Microsoft.Data.SqlClient.PerformanceTests.csproj`
-Configure `runnerconfig.json` file with connection string and preferred settings to run Benchmark Jobs.
+They can be run from the command line by following the instructions below.
+
+Launch a shell and change into the project directory:
+
+PowerShell:
+
+```pwsh
+> cd src\Microsoft.Data.SqlClient\tests\PerformanceTests
+```
+
+Bash:
+
+```bash
+$ cd src/Microsoft.Data.SqlClient/tests/PerformanceTests
+```
+
+### Create Database
+
+Create an empty database for the benchmarks to use.  This example assumes
+a local SQL server instance using SQL authentication:
 
 ```bash
-cd src\Microsoft.Data.SqlClient\tests\PerformanceTests
-dotnet run -c Release -f net8.0
+$ sqlcmd -S localhost -U sa -P password
+1> create database [sqlclient-perf-db]
+2> go
+1> quit
+```
+
+The default `runnerconfig.json` expects a database named `sqlclient-perf-db`,
+but you may change the config to use any existing database.  All tables in
+the database will be dropped when running the benchmarks.
+
+### Configure Runner
+
+Configure the benchmarks by editing the `runnerconfig.json` file directly in the
+`PerformanceTests` directory with an appropriate connection string and benchmark
+settings:
+
+```json
+{
+  "ConnectionString": "Server=tcp:localhost; Integrated Security=true; Initial Catalog=sqlclient-perf-db;",
+  "UseManagedSniOnWindows": false,
+  "Benchmarks":
+  {
+    "SqlConnectionRunnerConfig":
+    {
+      "Enabled": true,
+      "LaunchCount": 1,
+      "IterationCount": 50,
+      "InvocationCount":30,
+      "WarmupCount": 5,
+      "RowCount": 0
+    },
+    ...
+  }
+}
 ```
 
-_Only "**Release** Configuration" applies to Performance Tests_
+Individual benchmarks may be enabled or disabled, and each has several
+benchmarking options for fine tuning.
+
+After making edits to `runnerconfig.json` you must perform a build which will
+copy the file into the `artifacts` directory alongside the benchmark DLL.  By
+default, the benchmarks look for `runnerconfig.json` in the same directory as
+the DLL.
+
+Optionally, to avoid polluting your git workspace and requring a build after
+each config change, copy `runnerconfig.json` to a new file, make your edits
+there, and then specify the new file with the RUNNER_CONFIG environment
+variable.
+
+PowerShell:
+
+```pwsh
+> copy runnerconfig.json $HOME\.configs\runnerconfig.json
+
+# Make edits to $HOME\.configs\runnerconfig.json
+
+# You must set the RUNNER_CONFIG environment variable for the current shell.
+> $env:RUNNER_CONFIG="${HOME}\.configs\runnerconfig.json"
+```
+
+Bash:
+
+```bash
+$ cp runnerconfig.json ~/.configs/runnerconfig.json
+
+# Make edits to ~/.configs/runnerconfig.json
+
+# Optionally export RUNNER_CONFIG.
+$ export RUNNER_CONFIG=~/.configs/runnerconfig.json
+```
+
+### Run Benchmarks
+
+All benchmarks must be compiled and run in **Release** configuration.
+
+PowerShell:
+
+```pwsh
+> dotnet run -c Release -f net9.0
+```
+
+Bash:
+
+```bash
+# Omit RUNNER_CONFIG if you exported it earlier, or if you're using the
+# copy prepared by the build.
+$ dotnet run -c Release -f net9.0
+
+$ RUNNER_CONFIG=~/.configs/runnerconfig.json dotnet run -c Release -f net9.0
+```

src/Microsoft.Data.SqlClient/tests/PerformanceTests/BenchmarkRunners/BaseRunner.cs

@@ -2,17 +2,14 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-using System.IO;
-using Newtonsoft.Json;
-
 namespace Microsoft.Data.SqlClient.PerformanceTests
 {
     public abstract class BaseRunner
     {
         public BaseRunner()
         {
-            s_config = JsonConvert.DeserializeObject<Config>(File.ReadAllText("runnerconfig.json"));
-            s_datatypes = JsonConvert.DeserializeObject<DataTypes>(File.ReadAllText("datatypes.json"));
+            s_config = Config.Load();
+            s_datatypes = DataTypes.Load();
         }
 
         internal static Config s_config;

src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Config.cs

@@ -12,6 +12,28 @@ public class Config
         public string ConnectionString;
         public bool UseManagedSniOnWindows;
         public Benchmarks Benchmarks;
+
+        /// <summary>
+        /// Load the benchmark configuration from a JSON file.
+        ///
+        /// If the environment variable "RUNNER_CONFIG" is set, it will be used
+        /// as the path to the config file.  Otherwise, the file
+        /// "runnerconfig.json" in the current working directory will be used.
+        /// </summary>
+        ///
+        /// <returns>
+        ///   The Config instance populated from the JSON config file.
+        /// </returns>
+        ///
+        /// <exception cref="InvalidOperationException">
+        ///   Thrown if the config file cannot be read or deserialized.
+        /// </exception>
+        ///
+        public static Config Load()
+        {
+            return Loader.FromJsonFile<Config>(
+                "runnerconfig.json", "RUNNER_CONFIG");
+        }
     }
 
     public class Benchmarks

src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/DataTypes.cs

@@ -18,6 +18,28 @@ public class DataTypes
         public MaxLengthBinaryType[] Binary;
         public MaxLengthValueLengthType[] MaxTypes;
         public DataType[] Others;
+
+        /// <summary>
+        /// Load the data types configuration from a JSON file.
+        ///
+        /// If the environment variable "DATATYPES_CONFIG" is set, it will be
+        /// used as the path to the config file.  Otherwise, the file
+        /// "datatypes.json" in the current working directory will be used.
+        /// </summary>
+        ///
+        /// <returns>
+        ///   The DataTypes instance populated from the JSON file.
+        /// </returns>
+        ///
+        /// <exception cref="InvalidOperationException">
+        ///   Thrown if the config file cannot be read or deserialized.
+        /// </exception>
+        ///
+        public static DataTypes Load()
+        {
+            return Loader.FromJsonFile<DataTypes>(
+                "datatypes.json", "DATATYPES_CONFIG");
+        }
     }
 
     /// <summary>

src/Microsoft.Data.SqlClient/tests/PerformanceTests/Config/Loader.cs

@@ -0,0 +1,78 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+using System;
+using System.IO;
+using System.Text.Json;
+
+namespace Microsoft.Data.SqlClient.PerformanceTests
+{
+    public static class Loader
+    {
+        /// <summary>
+        ///   Load a JSON config file into the given type.
+        /// </summary>
+        ///
+        /// <typeparam name="T">
+        ///   The type to deserialize the JSON into.
+        /// </typeparam>
+        /// 
+        /// <param name="path">
+        ///   The path to the JSON config file.
+        /// </param>
+        /// 
+        /// <param name="envOverride">
+        ///   An optional environment variable that, if set, will be used as
+        ///   the config file path, ignoring path.
+        /// </param>
+        ///
+        /// <returns>
+        ///   The T instance populated from the JSON config file.
+        /// </returns>
+        /// 
+        /// <exception cref="InvalidOperationException">
+        ///   Thrown if the config file cannot be read or deserialized.
+        /// </exception>
+        ///
+        public static T FromJsonFile<T>(
+            string path,
+            string envOverride = null)
+            where T : class
+        {
+            string configFile =
+              envOverride is null
+              ? path
+              : Environment.GetEnvironmentVariable(envOverride)
+                ?? path;
+
+            T config = null;
+            Exception error = null;
+            try
+            {
+                using var stream = File.OpenRead(configFile);
+                config =
+                    JsonSerializer.Deserialize<T>(
+                        stream,
+                        new JsonSerializerOptions
+                        {
+                            IncludeFields = true,
+                            ReadCommentHandling = JsonCommentHandling.Skip
+                        });
+            }
+            catch (Exception ex)
+            {
+                error = ex;
+            }
+
+            if (config is null || error is not null)
+            {
+                throw new InvalidOperationException(
+                    $"Failed to load {typeof(T).Name} config from file=" +
+                    $"{configFile}", error);
+            }
+
+            return config;
+        }
+    }
+}

src/Microsoft.Data.SqlClient/tests/PerformanceTests/Microsoft.Data.SqlClient.PerformanceTests.csproj

@@ -3,46 +3,31 @@
     <OutputType>Exe</OutputType>
     <AssemblyName>PerformanceTests</AssemblyName>
     <TargetFrameworks>net8.0;net9.0</TargetFrameworks>
-    <EnableDefaultCompileItems>false</EnableDefaultCompileItems>
     <Configurations>Debug;Release;</Configurations>
     <IntermediateOutputPath>$(ObjFolder)$(Configuration).$(Platform).$(AssemblyName)</IntermediateOutputPath>
     <OutputPath>$(BinFolder)$(Configuration).$(Platform).$(AssemblyName)</OutputPath>
     <StartupObject>Microsoft.Data.SqlClient.PerformanceTests.Program</StartupObject>
   </PropertyGroup>
+
   <ItemGroup>
     <EmbeddedResource Remove="BenchmarkDotNet.Artifacts\**" />
     <None Remove="BenchmarkDotNet.Artifacts\**" />
   </ItemGroup>
+
   <ItemGroup>
     <None Remove=".AssemblyAttributes" />
   </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="$(NetCoreSource)src\Microsoft.Data.SqlClient.csproj" />
   </ItemGroup>
+
   <ItemGroup>
     <PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
-    <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
   </ItemGroup>
+
   <ItemGroup>
     <None Include="datatypes.json" CopyToOutputDirectory="PreserveNewest" />
     <None Include="runnerconfig.json" CopyToOutputDirectory="PreserveNewest" />
   </ItemGroup>
-  <ItemGroup>
-    <Compile Include="Config\Config.cs" />
-    <Compile Include="Config\Constants.cs" />
-    <Compile Include="Config\DataTypes.cs" />
-    <Compile Include="Config\BenchmarkConfig.cs" />
-    <Compile Include="DBFramework\Table.cs" />
-    <Compile Include="DBFramework\Column.cs" />
-    <Compile Include="DBFramework\DbUtils.cs" />
-    <Compile Include="DBFramework\TablePatterns.cs" />
-    <Compile Include="BenchmarkRunners\BaseRunner.cs" />
-    <Compile Include="BenchmarkRunners\SqlBulkCopyRunner.cs" />
-    <Compile Include="BenchmarkRunners\SqlConnectionRunner.cs" />
-    <Compile Include="BenchmarkRunners\SqlCommandRunner.cs" />
-    <Compile Include="BenchmarkRunners\DataTypeReaderRunner.cs" />
-    <Compile Include="BenchmarkRunners\DataTypeReaderAsyncRunner.cs" />
-    <Compile Include="Program.cs" />
-  </ItemGroup>
 </Project>