Skip to content

Commit 544a1c9

Browse files
psfinakipsfinaki
authored
Smoke test some benchmarks in CI (#16554)
* Smoke test benchmarks * Rename the file in the pipeline * I guess
1 parent 61a5792 commit 544a1c9

21 files changed

+128
-51
lines changed

‎FSharp.Benchmarks.sln‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
Microsoft Visual Studio Solution File, Format Version 12.00
1+
2+
Microsoft Visual Studio Solution File, Format Version 12.00
23
# Visual Studio Version 17
34
VisualStudioVersion = 17.1.32113.165
45
MinimumVisualStudioVersion = 10.0.40219.1
@@ -26,6 +27,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FCSSourceFiles", "tests\ben
2627
EndProject
2728
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Test.Utilities", "tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj", "{0B149238-0912-493E-8877-F831AE01B942}"
2829
EndProject
30+
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Benchmarks.Common", "tests\benchmarks\FSharp.Benchmarks.Common\FSharp.Benchmarks.Common.fsproj", "{62DED1EA-6A33-4537-8ED2-118462D0FEE5}"
31+
EndProject
2932
Global
3033
GlobalSection(SolutionConfigurationPlatforms) = preSolution
3134
Debug|Any CPU = Debug|Any CPU
@@ -109,6 +112,14 @@ Global
109112
{0B149238-0912-493E-8877-F831AE01B942}.Release|Any CPU.Build.0 = Release|Any CPU
110113
{0B149238-0912-493E-8877-F831AE01B942}.ReleaseCompressed|Any CPU.ActiveCfg = Debug|Any CPU
111114
{0B149238-0912-493E-8877-F831AE01B942}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
115+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
116+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.Debug|Any CPU.Build.0 = Debug|Any CPU
117+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
118+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.Proto|Any CPU.Build.0 = Debug|Any CPU
119+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.Release|Any CPU.ActiveCfg = Release|Any CPU
120+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.Release|Any CPU.Build.0 = Release|Any CPU
121+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.ReleaseCompressed|Any CPU.ActiveCfg = Release|Any CPU
122+
{62DED1EA-6A33-4537-8ED2-118462D0FEE5}.ReleaseCompressed|Any CPU.Build.0 = Release|Any CPU
112123
EndGlobalSection
113124
GlobalSection(SolutionProperties) = preSolution
114125
HideSolutionNode = FALSE

‎FSharp.Compiler.Service.sln‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,8 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.Interactive
6161
EndProject
6262
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Compiler.UnitTests", "tests\FSharp.Compiler.UnitTests\FSharp.Compiler.UnitTests.fsproj", "{0C0BDAF4-7D47-4BDA-9992-077F63D6B494}"
6363
EndProject
64+
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Benchmarks.Common", "tests\benchmarks\FSharp.Benchmarks.Common\FSharp.Benchmarks.Common.fsproj", "{A7ACFD1F-D1B8-483A-A210-200BB6B4BD7E}"
65+
EndProject
6466
Global
6567
GlobalSection(SolutionConfigurationPlatforms) = preSolution
6668
Debug|Any CPU = Debug|Any CPU
@@ -143,6 +145,10 @@ Global
143145
{0C0BDAF4-7D47-4BDA-9992-077F63D6B494}.Debug|Any CPU.Build.0 = Debug|Any CPU
144146
{0C0BDAF4-7D47-4BDA-9992-077F63D6B494}.Release|Any CPU.ActiveCfg = Release|Any CPU
145147
{0C0BDAF4-7D47-4BDA-9992-077F63D6B494}.Release|Any CPU.Build.0 = Release|Any CPU
148+
{A7ACFD1F-D1B8-483A-A210-200BB6B4BD7E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
149+
{A7ACFD1F-D1B8-483A-A210-200BB6B4BD7E}.Debug|Any CPU.Build.0 = Debug|Any CPU
150+
{A7ACFD1F-D1B8-483A-A210-200BB6B4BD7E}.Release|Any CPU.ActiveCfg = Release|Any CPU
151+
{A7ACFD1F-D1B8-483A-A210-200BB6B4BD7E}.Release|Any CPU.Build.0 = Release|Any CPU
146152
EndGlobalSection
147153
GlobalSection(SolutionProperties) = preSolution
148154
HideSolutionNode = FALSE
@@ -154,6 +160,7 @@ Global
154160
{35F5F1C5-AE4F-4B5A-8D94-1AF708724FD5} = {AF321816-B4A0-41DD-9A1D-484E8A20C6F6}
155161
{C1950E28-1CB7-4DEC-BB3A-8A0443A17282} = {AF321816-B4A0-41DD-9A1D-484E8A20C6F6}
156162
{07CD957A-3C31-4F75-A735-16CE72E1BD71} = {AF321816-B4A0-41DD-9A1D-484E8A20C6F6}
163+
{A7ACFD1F-D1B8-483A-A210-200BB6B4BD7E} = {AF321816-B4A0-41DD-9A1D-484E8A20C6F6}
157164
EndGlobalSection
158165
GlobalSection(ExtensibilityGlobals) = postSolution
159166
SolutionGuid = {F9A60F3B-D894-4C8E-BA0F-C51115B25A5A}

‎FSharp.sln‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
106106
src\Compiler\FSCompCheck.fsx = src\Compiler\FSCompCheck.fsx
107107
EndProjectSection
108108
EndProject
109+
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Benchmarks.Common", "tests\benchmarks\FSharp.Benchmarks.Common\FSharp.Benchmarks.Common.fsproj", "{7D482560-DF6F-46A5-B50C-20ECF7C38759}"
110+
EndProject
109111
Global
110112
GlobalSection(SolutionConfigurationPlatforms) = preSolution
111113
Debug|Any CPU = Debug|Any CPU
@@ -416,6 +418,18 @@ Global
416418
{9C7523BA-7AB2-4604-A5FD-653E82C2BAD1}.Release|Any CPU.Build.0 = Release|Any CPU
417419
{9C7523BA-7AB2-4604-A5FD-653E82C2BAD1}.Release|x86.ActiveCfg = Release|Any CPU
418420
{9C7523BA-7AB2-4604-A5FD-653E82C2BAD1}.Release|x86.Build.0 = Release|Any CPU
421+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
422+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Debug|Any CPU.Build.0 = Debug|Any CPU
423+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Debug|x86.ActiveCfg = Debug|Any CPU
424+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Debug|x86.Build.0 = Debug|Any CPU
425+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
426+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Proto|Any CPU.Build.0 = Debug|Any CPU
427+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Proto|x86.ActiveCfg = Debug|Any CPU
428+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Proto|x86.Build.0 = Debug|Any CPU
429+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Release|Any CPU.ActiveCfg = Release|Any CPU
430+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Release|Any CPU.Build.0 = Release|Any CPU
431+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Release|x86.ActiveCfg = Release|Any CPU
432+
{7D482560-DF6F-46A5-B50C-20ECF7C38759}.Release|x86.Build.0 = Release|Any CPU
419433
EndGlobalSection
420434
GlobalSection(SolutionProperties) = preSolution
421435
HideSolutionNode = FALSE
@@ -447,6 +461,7 @@ Global
447461
{209C7D37-8C01-413C-8698-EC25F4C86976} = {B8DDA694-7939-42E3-95E5-265C2217C142}
448462
{BEC6E796-7E53-4888-AAFC-B8FD55C425DF} = {CE70D631-C5DC-417E-9CDA-B16097BEF1AC}
449463
{9C7523BA-7AB2-4604-A5FD-653E82C2BAD1} = {CE70D631-C5DC-417E-9CDA-B16097BEF1AC}
464+
{7D482560-DF6F-46A5-B50C-20ECF7C38759} = {CE70D631-C5DC-417E-9CDA-B16097BEF1AC}
450465
EndGlobalSection
451466
GlobalSection(ExtensibilityGlobals) = postSolution
452467
SolutionGuid = {BD5177C7-1380-40E7-94D2-7768E1A8B1B8}

‎VisualFSharp.sln‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Editor.Tests", "vsin
193193
EndProject
194194
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FSharp.Editor.IntegrationTests", "vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj", "{E31F9B59-FCF1-4D04-8762-C7BB60285A7B}"
195195
EndProject
196+
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Benchmarks.Common", "tests\benchmarks\FSharp.Benchmarks.Common\FSharp.Benchmarks.Common.fsproj", "{6734FC6F-B5F3-45E1-9A72-720378BB49C9}"
197+
EndProject
196198
Global
197199
GlobalSection(SolutionConfigurationPlatforms) = preSolution
198200
Debug|Any CPU = Debug|Any CPU
@@ -1019,6 +1021,18 @@ Global
10191021
{E31F9B59-FCF1-4D04-8762-C7BB60285A7B}.Release|Any CPU.Build.0 = Release|Any CPU
10201022
{E31F9B59-FCF1-4D04-8762-C7BB60285A7B}.Release|x86.ActiveCfg = Release|Any CPU
10211023
{E31F9B59-FCF1-4D04-8762-C7BB60285A7B}.Release|x86.Build.0 = Release|Any CPU
1024+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1025+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Debug|Any CPU.Build.0 = Debug|Any CPU
1026+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Debug|x86.ActiveCfg = Debug|Any CPU
1027+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Debug|x86.Build.0 = Debug|Any CPU
1028+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
1029+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Proto|Any CPU.Build.0 = Debug|Any CPU
1030+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Proto|x86.ActiveCfg = Debug|Any CPU
1031+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Proto|x86.Build.0 = Debug|Any CPU
1032+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Release|Any CPU.ActiveCfg = Release|Any CPU
1033+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Release|Any CPU.Build.0 = Release|Any CPU
1034+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Release|x86.ActiveCfg = Release|Any CPU
1035+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9}.Release|x86.Build.0 = Release|Any CPU
10221036
EndGlobalSection
10231037
GlobalSection(SolutionProperties) = preSolution
10241038
HideSolutionNode = FALSE
@@ -1099,6 +1113,7 @@ Global
10991113
{39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B}
11001114
{CBC96CC7-65AB-46EA-A82E-F6A788DABF80} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}
11011115
{E31F9B59-FCF1-4D04-8762-C7BB60285A7B} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}
1116+
{6734FC6F-B5F3-45E1-9A72-720378BB49C9} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B}
11021117
EndGlobalSection
11031118
GlobalSection(ExtensibilityGlobals) = postSolution
11041119
SolutionGuid = {48EDBBBE-C8EE-4E3C-8B19-97184A487B37}

‎azure-pipelines.yml‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -713,14 +713,14 @@ stages:
713713
continueOnError: true
714714
condition: always()
715715

716-
# Build benchmarks
717-
- job: Plain_Build_Benchmarks
716+
# Build and run fast benchmarks
717+
- job: Benchmarks
718718
pool:
719719
name: $(DncEngPublicBuildPool)
720720
demands: ImageOverride -equals $(WindowsMachineQueueName)
721721
variables:
722722
- name: _BuildConfig
723-
value: Debug
723+
value: Release
724724
steps:
725725
- checkout: self
726726
clean: true
@@ -734,11 +734,15 @@ stages:
734734
includePreviewVersions: true
735735
workingDirectory: $(Build.SourcesDirectory)
736736
installationPath: $(Agent.ToolsDirectory)/dotnet
737-
- script: dotnet build .\FSharp.Benchmarks.sln /bl:\"artifacts/log/$(_BuildConfig)/BenchmarkBuild.binlog\"
737+
- script: dotnet build -c $(_BuildConfig) .\FSharp.Benchmarks.sln /bl:\"artifacts/log/$(_BuildConfig)/BenchmarkBuild.binlog\"
738738
workingDirectory: $(Build.SourcesDirectory)
739-
displayName: Regular rebuild of FSharp.Benchmarks.sln
739+
displayName: Plain build of FSharp.Benchmarks.sln
740740
continueOnError: true
741741
condition: always()
742+
- script: .\Build.cmd -c $(_BuildConfig)
743+
- pwsh: ./SmokeTestBenchmarks.ps1
744+
workingDirectory: $(Build.SourcesDirectory)/tests/benchmarks
745+
displayName: Smoke test fast benchmarks
742746

743747
# Test trimming on Windows
744748
- job: Build_And_Test_Trimming_Windows

‎buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
FSharp.Build.UnitTests.dll
2+
FSharp.Benchmarks.Common.dll
23
FSharp.Compiler.Benchmarks.dll
34
FSharp.Compiler.ComponentTests.dll
45
FSharp.Test.Utilities.dll

‎tests/benchmarks/FCSBenchmarks/BenchmarkComparison/HistoricalBenchmark.fsproj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,9 @@
5959
<HintPath>$(FSharpCoreDllPath)</HintPath>
6060
</Reference>
6161
</ItemGroup>
62+
<ItemGroup>
63+
<ProjectReference Include="..\..\FSharp.Benchmarks.Common\FSharp.Benchmarks.Common.fsproj" />
64+
</ItemGroup>
6265

6366
<Target Name="FakeBuild" BeforeTargets="Build">
6467
<Message Text="Type=$(FcsReferenceType) FcsDllPath=$(FcsDllPath)" />

‎tests/benchmarks/FCSBenchmarks/BenchmarkComparison/Program.fs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ namespace HistoricalBenchmark
33
open System.IO
44
open BenchmarkDotNet.Attributes
55
open BenchmarkDotNet.Running
6+
open FSharp.Benchmarks.Common.Categories
67

78
[<AbstractClass>]
89
type SingleFileCompilerBenchmarkBase(compiler : SingleFileCompiler) =
@@ -20,6 +21,7 @@ type SingleFileCompilerBenchmarkBase(compiler : SingleFileCompiler) =
2021

2122
[<MemoryDiagnoser>]
2223
[<JsonExporter>]
24+
[<BenchmarkCategory(ShortCategory)>]
2325
type DecentlySizedStandAloneFileBenchmark() =
2426
inherit SingleFileCompilerBenchmarkBase(
2527
SingleFileCompiler(

‎tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/BackgroundCompilerBenchmarks.fs‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,10 @@ open FSharp.Compiler.CodeAnalysis
66
open FSharp.Compiler.Text
77
open FSharp.Compiler.Diagnostics
88
open FSharp.Test.ProjectGeneration
9-
open BenchmarkDotNet.Engines
10-
11-
12-
[<Literal>]
13-
let FSharpCategory = "fsharp"
14-
9+
open FSharp.Benchmarks.Common.Categories
1510

1611
[<MemoryDiagnoser>]
17-
[<BenchmarkCategory(FSharpCategory)>]
12+
[<BenchmarkCategory(LongCategory)>]
1813
type BackgroundCompilerBenchmarks () =
1914

2015
let size = 50
@@ -104,7 +99,7 @@ type BackgroundCompilerBenchmarks () =
10499
this.Benchmark.DeleteProjectDir()
105100

106101
[<MemoryDiagnoser>]
107-
[<BenchmarkCategory(FSharpCategory)>]
102+
[<BenchmarkCategory(ShortCategory)>]
108103
type ParsingBenchmark() =
109104

110105
let mutable checker: FSharpChecker = Unchecked.defaultof<_>
@@ -134,7 +129,7 @@ type ParsingBenchmark() =
134129
failwith "ParseHadErrors"
135130

136131
[<MemoryDiagnoser>]
137-
[<BenchmarkCategory(FSharpCategory)>]
132+
[<BenchmarkCategory(LongCategory)>]
138133
type NoFileSystemCheckerBenchmark() =
139134

140135
let size = 30
@@ -229,8 +224,8 @@ type TestProjectType =
229224

230225
[<MemoryDiagnoser>]
231226
[<ThreadingDiagnoser>]
232-
[<BenchmarkCategory(FSharpCategory)>]
233227
[<SimpleJob(warmupCount=1,iterationCount=4)>]
228+
[<BenchmarkCategory(ShortCategory)>]
234229
type TransparentCompilerBenchmark() =
235230

236231
let size = 30
@@ -331,9 +326,9 @@ type TransparentCompilerBenchmark() =
331326
benchmark.DeleteProjectDir()
332327

333328

329+
// needs Giraffe repo somewhere nearby, hence benchmarks disabled
334330
[<MemoryDiagnoser>]
335331
[<ThreadingDiagnoser>]
336-
[<BenchmarkCategory(FSharpCategory)>]
337332
[<SimpleJob(warmupCount=1,iterationCount=8)>]
338333
type TransparentCompilerGiraffeBenchmark() =
339334

@@ -399,7 +394,7 @@ type TransparentCompilerGiraffeBenchmark() =
399394
checkFile (this.Project.SourceFiles |> List.last).Id expectOk
400395
}
401396

402-
[<Benchmark>]
397+
// [<Benchmark>]
403398
member this.SomeWorkflow() =
404399

405400
use _ = Activity.start "Benchmark" [

‎tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/CompilerServiceBenchmarks.fs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ open FSharp.Compiler.AbstractIL.ILBinaryReader
1212
open BenchmarkDotNet.Attributes
1313
open FSharp.Compiler.Benchmarks
1414
open Microsoft.CodeAnalysis.Text
15+
open FSharp.Benchmarks.Common.Categories
1516

1617
type private Config =
1718
{
@@ -66,6 +67,7 @@ let function%s{moduleName} (x: %s{moduleName}) =
6667

6768

6869
[<MemoryDiagnoser>]
70+
[<BenchmarkCategory(ShortCategory)>]
6971
type CompilerServiceBenchmarks() =
7072
let mutable configOpt : Config option = None
7173
let sourcePath = Path.Combine(__SOURCE_DIRECTORY__, "../decentlySizedStandAloneFile.fs")

0 commit comments

Comments
 (0)