ci/performance testing (microsoft#2304)

* ci: performance project scaffolding
* chore: moves benchmark to dedicated directory
* chore: commits initial benchmark results
* ci: adds a comparison project for benchmark results
* ci: adds benchmark runs and comparison
* ci: renames empty models tests
* chore: refactors to a command structure
* chore: moves models to a dedicated source file
* chore: uses STJ attribute based serialization
* ci: fixes missing project argument
* ci: fixes reports path
* chore: moves policies to own directory
* chore: no warn on static method for perf test project
* chore: linting
* chore: reduces run time for empty performance tests
* chore: linting
* fix: a bug where the input stream would be disposed not matter what
* ci: fixes working directory for benchmark run
* ci: adds basic infrastructure for descriptions based performance tests
* chore: refresh empty object tests
* ci: adds comparison run for descriptions comparisons
* ci: adds percentage based comparisons
* ci: adds large description to performance tests
* ci: adds yaml descriptions perf tests to compare
* ci: adds truncation in percentage difference to reduce noise
---------

Signed-off-by: Vincent Biret <vibiret@microsoft.com>

Author: baywet

.github/workflows/ci-cd.yml

@@ -16,7 +16,7 @@ jobs:
       - name: Setup .NET
         uses: actions/setup-dotnet@v4
         with:
-          dotnet-version: 8.0.x
+          dotnet-version: 8.x
 
       - name: Checkout repository
         id: checkout_repo
@@ -51,3 +51,40 @@ jobs:
       - name: Validate Trimming warnings
         run: dotnet publish -c Release -r win-x64 /p:TreatWarningsAsErrors=true /warnaserror -f net8.0
         working-directory: ./test/Microsoft.OpenApi.Trimming.Tests
+
+  validate-performance:
+    name: Validate performance of the library
+    runs-on: ubuntu-latest
+    needs: [ci]
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup .NET
+        uses: actions/setup-dotnet@v4
+        with:
+          dotnet-version: 8.x
+
+      - name: Copy committed results
+        run: |
+          mkdir -p ./performanceResults
+          cp -r ./performance/benchmark/BenchmarkDotNet.Artifacts/results/* ./performanceResults
+      
+      - name: Run performance tests
+        run: |
+          dotnet run -c Release
+        working-directory: ./performance/benchmark
+
+      - name: Run comparison tool for empty models
+        run: dotnet run -c Release --project ./performance/resultsComparer/resultsComparer.csproj -- compare $OLD_REPORT $NEW_REPORT -p IdenticalMemoryUsage
+        shell: bash
+        env:
+          NEW_REPORT: "${{ github.workspace }}/performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.EmptyModels-report.json"
+          OLD_REPORT: "${{ github.workspace }}/performanceResults/performance.EmptyModels-report.json"
+
+      - name: Run comparison tool for descriptions
+        run: dotnet run -c Release --project ./performance/resultsComparer/resultsComparer.csproj -- compare $OLD_REPORT $NEW_REPORT -p ZeroPointTwoPercentDifferenceMemoryUsage
+        shell: bash
+        env:
+          NEW_REPORT: "${{ github.workspace }}/performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.json"
+          OLD_REPORT: "${{ github.workspace }}/performanceResults/performance.Descriptions-report.json"

.vscode/launch.json

@@ -19,6 +19,37 @@
             "console": "internalConsole",
             "stopAtEntry": false
         },
+        {
+            // Use IntelliSense to find out which attributes exist for C# debugging
+            // Use hover for the description of the existing attributes
+            // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/main/debugger-launchjson.md
+            "name": "Launch Benchmark",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceFolder}/performance/benchmark/bin/Release/net8.0/PerformanceTests.dll",
+            // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
+            "console": "internalConsole",
+            "stopAtEntry": false,
+        },
+        {
+            // Use IntelliSense to find out which attributes exist for C# debugging
+            // Use hover for the description of the existing attributes
+            // For further information visit https://github.com/OmniSharp/omnisharp-vscode/blob/main/debugger-launchjson.md
+            "name": "Launch Results comparer",
+            "type": "coreclr",
+            "request": "launch",
+            "preLaunchTask": "build",
+            // If you have changed target frameworks, make sure to update the program path.
+            "program": "${workspaceFolder}/performance/resultsComparer/bin/Debug/net8.0/resultsComparer.dll",
+            "cwd": "${workspaceFolder}/performance/resultsComparer",
+            "args": ["compare"],
+            // For more information about the 'console' field, see https://aka.ms/VSCode-CS-LaunchJson-Console
+            "console": "internalConsole",
+            "stopAtEntry": false,
+            "requireExactSource": false,
+        },
         {
             // Use IntelliSense to find out which attributes exist for C# debugging
             // Use hover for the description of the existing attributes

.vscode/tasks.json

@@ -10,7 +10,7 @@
             "group": "build",
             "args": [
                 "build",
-                "${workspaceFolder}/src/Microsoft.OpenApi.Hidi/Microsoft.OpenApi.Hidi.csproj",
+                "${workspaceFolder}/Microsoft.OpenApi.sln",
                 "/property:GenerateFullPaths=true",
                 "/consoleloggerparameters:NoSummary"
             ],

Microsoft.OpenApi.sln

@@ -30,6 +30,12 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.OpenApi.Hidi", "s
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.OpenApi.Hidi.Tests", "test\Microsoft.OpenApi.Hidi.Tests\Microsoft.OpenApi.Hidi.Tests.csproj", "{6ADC5D41-EDD2-4206-B815-5DFF739C6832}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PerformanceTests", "performance\benchmark\PerformanceTests.csproj", "{537E49E3-325E-40EE-A90E-7556D4D333AA}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "performance", "performance", "{4BB7E3F7-CA7E-45D3-B5AC-5DBB510FD528}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "resultsComparer", "performance\resultsComparer\resultsComparer.csproj", "{5EEA836B-3E08-4BE1-82B8-5236D031B497}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -68,6 +74,14 @@ Global
 		{6ADC5D41-EDD2-4206-B815-5DFF739C6832}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{6ADC5D41-EDD2-4206-B815-5DFF739C6832}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{6ADC5D41-EDD2-4206-B815-5DFF739C6832}.Release|Any CPU.Build.0 = Release|Any CPU
+		{537E49E3-325E-40EE-A90E-7556D4D333AA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{537E49E3-325E-40EE-A90E-7556D4D333AA}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{537E49E3-325E-40EE-A90E-7556D4D333AA}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{537E49E3-325E-40EE-A90E-7556D4D333AA}.Release|Any CPU.Build.0 = Release|Any CPU
+		{5EEA836B-3E08-4BE1-82B8-5236D031B497}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{5EEA836B-3E08-4BE1-82B8-5236D031B497}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{5EEA836B-3E08-4BE1-82B8-5236D031B497}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{5EEA836B-3E08-4BE1-82B8-5236D031B497}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -81,6 +95,7 @@ Global
 		{1D2E0C6E-B103-4CB6-912E-D56FA1501296} = {6357D7FD-2DE4-4900-ADB9-ABC37052040A}
 		{538936B4-5E14-4EA3-9FD0-F43E2DD014FB} = {E546B92F-20A8-49C3-8323-4B25BB78F3E1}
 		{6ADC5D41-EDD2-4206-B815-5DFF739C6832} = {6357D7FD-2DE4-4900-ADB9-ABC37052040A}
+		{5EEA836B-3E08-4BE1-82B8-5236D031B497} = {4BB7E3F7-CA7E-45D3-B5AC-5DBB510FD528}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {9F171EFC-0DB5-4B10-ABFA-AF48D52CC565}

performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report-github.md

@@ -0,0 +1,18 @@
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.3476)
+11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores
+.NET SDK 8.0.408
+  [Host]   : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
+  ShortRun : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
+
+Job=ShortRun  IterationCount=3  LaunchCount=1  
+WarmupCount=3  
+
+```
+| Method       | Mean         | Error         | StdDev       | Gen0       | Gen1       | Gen2      | Allocated    |
+|------------- |-------------:|--------------:|-------------:|-----------:|-----------:|----------:|-------------:|
+| PetStoreYaml |     450.5 μs |      59.26 μs |      3.25 μs |    58.5938 |    11.7188 |         - |    377.15 KB |
+| PetStoreJson |     172.8 μs |     123.46 μs |      6.77 μs |    39.0625 |     7.8125 |         - |    239.29 KB |
+| GHESYaml     | 943,452.7 μs | 137,685.49 μs |  7,547.01 μs | 66000.0000 | 21000.0000 | 3000.0000 | 389463.91 KB |
+| GHESJson     | 468,401.8 μs | 300,711.80 μs | 16,483.03 μs | 41000.0000 | 15000.0000 | 3000.0000 | 250934.62 KB |

performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.csv

@@ -0,0 +1,5 @@
+Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,LargeAddressAware,Platform,PowerPlanMode,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,NuGetReferences,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MemoryRandomization,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev,Gen0,Gen1,Gen2,Allocated
+PetStoreYaml,ShortRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,3,Default,1,Default,Default,Default,Default,Default,Default,16,3,450.5 μs,59.26 μs,3.25 μs,58.5938,11.7188,0.0000,377.15 KB
+PetStoreJson,ShortRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,3,Default,1,Default,Default,Default,Default,Default,Default,16,3,172.8 μs,123.46 μs,6.77 μs,39.0625,7.8125,0.0000,239.29 KB
+GHESYaml,ShortRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,3,Default,1,Default,Default,Default,Default,Default,Default,16,3,"943,452.7 μs","137,685.49 μs","7,547.01 μs",66000.0000,21000.0000,3000.0000,389463.91 KB
+GHESJson,ShortRun,False,Default,Default,Default,Default,Default,Default,11111111,Empty,RyuJit,Default,X64,8c5e7fda-e8bf-4a96-9a85-a6e23a8c635c,.NET 8.0,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,Default,Default,3,Default,1,Default,Default,Default,Default,Default,Default,16,3,"468,401.8 μs","300,711.80 μs","16,483.03 μs",41000.0000,15000.0000,3000.0000,250934.62 KB

performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.html

@@ -0,0 +1,35 @@
+<!DOCTYPE html>
+<html lang='en'>
+<head>
+<meta charset='utf-8' />
+<title>performance.Descriptions-20250409-150544</title>
+
+<style type="text/css">
+	table { border-collapse: collapse; display: block; width: 100%; overflow: auto; }
+	td, th { padding: 6px 13px; border: 1px solid #ddd; text-align: right; }
+	tr { background-color: #fff; border-top: 1px solid #ccc; }
+	tr:nth-child(even) { background: #f8f8f8; }
+</style>
+</head>
+<body>
+<pre><code>
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.3476)
+11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores
+.NET SDK 8.0.408
+  [Host]   : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
+  ShortRun : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
+</code></pre>
+<pre><code>Job=ShortRun  IterationCount=3  LaunchCount=1  
+WarmupCount=3  
+</code></pre>
+
+<table>
+<thead><tr><th>Method</th><th>Mean  </th><th>Error  </th><th>StdDev</th><th>Gen0</th><th>Gen1</th><th>Gen2</th><th>Allocated</th>
+</tr>
+</thead><tbody><tr><td>PetStoreYaml</td><td>450.5 &mu;s</td><td>59.26 &mu;s</td><td>3.25 &mu;s</td><td>58.5938</td><td>11.7188</td><td>-</td><td>377.15 KB</td>
+</tr><tr><td>PetStoreJson</td><td>172.8 &mu;s</td><td>123.46 &mu;s</td><td>6.77 &mu;s</td><td>39.0625</td><td>7.8125</td><td>-</td><td>239.29 KB</td>
+</tr><tr><td>GHESYaml</td><td>943,452.7 &mu;s</td><td>137,685.49 &mu;s</td><td>7,547.01 &mu;s</td><td>66000.0000</td><td>21000.0000</td><td>3000.0000</td><td>389463.91 KB</td>
+</tr><tr><td>GHESJson</td><td>468,401.8 &mu;s</td><td>300,711.80 &mu;s</td><td>16,483.03 &mu;s</td><td>41000.0000</td><td>15000.0000</td><td>3000.0000</td><td>250934.62 KB</td>
+</tr></tbody></table>
+</body>
+</html>

performance/benchmark/BenchmarkDotNet.Artifacts/results/performance.Descriptions-report.json

@@ -0,0 +1,42 @@
+```
+
+BenchmarkDotNet v0.14.0, Windows 11 (10.0.26100.3476)
+11th Gen Intel Core i7-1185G7 3.00GHz, 1 CPU, 8 logical and 4 physical cores
+.NET SDK 8.0.408
+  [Host]   : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
+  ShortRun : .NET 8.0.15 (8.0.1525.16413), X64 RyuJIT AVX-512F+CD+BW+DQ+VL+VBMI
+
+Job=ShortRun  IterationCount=3  LaunchCount=1  
+WarmupCount=3  
+
+```
+| Method                      | Mean       | Error       | StdDev     | Gen0   | Gen1   | Allocated |
+|---------------------------- |-----------:|------------:|-----------:|-------:|-------:|----------:|
+| EmptyApiCallback            |   4.557 ns |   2.2807 ns |  0.1250 ns | 0.0051 |      - |      32 B |
+| EmptyApiComponents          |   5.116 ns |   0.6915 ns |  0.0379 ns | 0.0166 |      - |     104 B |
+| EmptyApiContact             |   3.759 ns |   3.1240 ns |  0.1712 ns | 0.0076 |      - |      48 B |
+| EmptyApiDiscriminator       |   3.442 ns |   2.3747 ns |  0.1302 ns | 0.0064 |      - |      40 B |
+| EmptyDocument               | 397.830 ns |  46.9921 ns |  2.5758 ns | 0.1807 | 0.0005 |    1136 B |
+| EmptyApiEncoding            |   3.879 ns |   3.0270 ns |  0.1659 ns | 0.0089 |      - |      56 B |
+| EmptyApiExample             |   4.045 ns |   6.0543 ns |  0.3319 ns | 0.0089 |      - |      56 B |
+| EmptyApiExternalDocs        |   3.455 ns |   2.1233 ns |  0.1164 ns | 0.0064 |      - |      40 B |
+| EmptyApiHeader              |   4.633 ns |   3.8933 ns |  0.2134 ns | 0.0127 |      - |      80 B |
+| EmptyApiInfo                |   4.462 ns |   4.0279 ns |  0.2208 ns | 0.0127 |      - |      80 B |
+| EmptyApiLicense             |   3.670 ns |   1.6839 ns |  0.0923 ns | 0.0076 |      - |      48 B |
+| EmptyApiLink                |   4.388 ns |   1.9826 ns |  0.1087 ns | 0.0115 |      - |      72 B |
+| EmptyApiMediaType           |   3.857 ns |   1.4731 ns |  0.0807 ns | 0.0089 |      - |      56 B |
+| EmptyApiOAuthFlow           |   3.810 ns |   1.1359 ns |  0.0623 ns | 0.0089 |      - |      56 B |
+| EmptyApiOAuthFlows          |   3.979 ns |   5.5181 ns |  0.3025 ns | 0.0089 |      - |      56 B |
+| EmptyApiOperation           |  72.530 ns | 230.3314 ns | 12.6252 ns | 0.0599 | 0.0001 |     376 B |
+| EmptyApiParameter           |   4.919 ns |   3.2142 ns |  0.1762 ns | 0.0153 |      - |      96 B |
+| EmptyApiPathItem            |   3.966 ns |   0.7140 ns |  0.0391 ns | 0.0102 |      - |      64 B |
+| EmptyApiPaths               |  56.222 ns |  32.3248 ns |  1.7718 ns | 0.0395 |      - |     248 B |
+| EmptyApiRequestBody         |   3.683 ns |   2.3246 ns |  0.1274 ns | 0.0076 |      - |      48 B |
+| EmptyApiResponse            |   3.864 ns |   0.9334 ns |  0.0512 ns | 0.0089 |      - |      56 B |
+| EmptyApiResponses           |  49.325 ns |   7.2131 ns |  0.3954 ns | 0.0395 |      - |     248 B |
+| EmptyApiSchema              |  12.565 ns |   2.0834 ns |  0.1142 ns | 0.0650 |      - |     408 B |
+| EmptyApiSecurityRequirement |   8.411 ns |   1.5393 ns |  0.0844 ns | 0.0166 |      - |     104 B |
+| EmptyApiSecurityScheme      |   4.719 ns |   3.8028 ns |  0.2084 ns | 0.0140 |      - |      88 B |
+| EmptyApiServer              |   3.626 ns |   0.4928 ns |  0.0270 ns | 0.0076 |      - |      48 B |
+| EmptyApiServerVariable      |   3.589 ns |   0.2983 ns |  0.0164 ns | 0.0076 |      - |      48 B |
+| EmptyApiTag                 |   3.889 ns |   7.4113 ns |  0.4062 ns | 0.0076 |      - |      48 B |