Skip to content

Commit 5d14be8

Browse files
psfinakiKevinRansom
psfinaki
and
KevinRansom
authored
Build benchmarks in CI (#16518)
* Remove profiling startpoint project * Add bench build job * Up * up * up --------- Co-authored-by: Kevin Ransom (msft) <codecutter@hotmail.com>
1 parent ebbfeed commit 5d14be8

File tree

11 files changed

+92
-106
lines changed

11 files changed

+92
-106
lines changed

‎Directory.Build.props

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,10 @@
1616
<BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>
1717
</PropertyGroup>
1818

19+
<PropertyGroup Condition="'$(SolutionName)' == 'FSharp.Benchmarks' and '$(BUILDING_USING_DOTNET)' != 'false'">
20+
<BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>
21+
</PropertyGroup>
22+
1923
<!-- Temporary workaround if proj-info is used, allows Ionide's evaluation to work correctly. -->
2024
<PropertyGroup Condition="'$(BUILDING_USING_DOTNET)' != 'false' and '$(DotnetProjInfo)' == 'true'">
2125
<BUILDING_USING_DOTNET>true</BUILDING_USING_DOTNET>

‎FSharp.Benchmarks.sln

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,6 @@ Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Compiler.Benchmarks"
2424
EndProject
2525
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FCSSourceFiles", "tests\benchmarks\FCSBenchmarks\FCSSourceFiles\FCSSourceFiles.fsproj", "{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}"
2626
EndProject
27-
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Fsharp.ProfilingStartpointProject", "tests\benchmarks\Fsharp.ProfilingStartpointProject\Fsharp.ProfilingStartpointProject.fsproj", "{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}"
28-
EndProject
2927
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharp.Test.Utilities", "tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj", "{0B149238-0912-493E-8877-F831AE01B942}"
3028
EndProject
3129
Global
@@ -105,12 +103,6 @@ Global
105103
{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}.Release|Any CPU.Build.0 = Release|Any CPU
106104
{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}.ReleaseCompressed|Any CPU.ActiveCfg = Debug|Any CPU
107105
{0E2A7B27-3AD3-4C1D-BA0D-008A1200946F}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
108-
{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
109-
{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Debug|Any CPU.Build.0 = Debug|Any CPU
110-
{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Release|Any CPU.ActiveCfg = Release|Any CPU
111-
{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Release|Any CPU.Build.0 = Release|Any CPU
112-
{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.ReleaseCompressed|Any CPU.ActiveCfg = Debug|Any CPU
113-
{9F27346B-2FC6-4FD5-A932-4E80F331E6D6}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
114106
{0B149238-0912-493E-8877-F831AE01B942}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
115107
{0B149238-0912-493E-8877-F831AE01B942}.Debug|Any CPU.Build.0 = Debug|Any CPU
116108
{0B149238-0912-493E-8877-F831AE01B942}.Release|Any CPU.ActiveCfg = Release|Any CPU

‎VisualFSharp.sln

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -189,8 +189,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "FCSBenchmarks", "FCSBenchma
189189
tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1 = tests\benchmarks\FCSBenchmarks\SmokeTestAllBenchmarks.ps1
190190
EndProjectSection
191191
EndProject
192-
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Fsharp.ProfilingStartpointProject", "tests\benchmarks\Fsharp.ProfilingStartpointProject\Fsharp.ProfilingStartpointProject.fsproj", "{FE23BB65-276A-4E41-8CC7-F7752241DEBA}"
193-
EndProject
194192
Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "FSharp.Editor.Tests", "vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj", "{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}"
195193
EndProject
196194
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FSharp.Editor.IntegrationTests", "vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj", "{E31F9B59-FCF1-4D04-8762-C7BB60285A7B}"
@@ -997,18 +995,6 @@ Global
997995
{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|Any CPU.Build.0 = Release|Any CPU
998996
{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.ActiveCfg = Release|Any CPU
999997
{583182E1-3484-4A8F-AC06-7C0D232C0CA4}.Release|x86.Build.0 = Release|Any CPU
1000-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1001-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|Any CPU.Build.0 = Debug|Any CPU
1002-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.ActiveCfg = Debug|Any CPU
1003-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Debug|x86.Build.0 = Debug|Any CPU
1004-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.ActiveCfg = Debug|Any CPU
1005-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|Any CPU.Build.0 = Debug|Any CPU
1006-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.ActiveCfg = Debug|Any CPU
1007-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Proto|x86.Build.0 = Debug|Any CPU
1008-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.ActiveCfg = Release|Any CPU
1009-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|Any CPU.Build.0 = Release|Any CPU
1010-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.ActiveCfg = Release|Any CPU
1011-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA}.Release|x86.Build.0 = Release|Any CPU
1012998
{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
1013999
{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}.Debug|Any CPU.Build.0 = Debug|Any CPU
10141000
{CBC96CC7-65AB-46EA-A82E-F6A788DABF80}.Debug|x86.ActiveCfg = Debug|Any CPU
@@ -1111,7 +1097,6 @@ Global
11111097
{EB015235-1E07-4CDA-9CC6-3FBCC27910D1} = {B8DDA694-7939-42E3-95E5-265C2217C142}
11121098
{583182E1-3484-4A8F-AC06-7C0D232C0CA4} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
11131099
{39CDF34B-FB23-49AE-AB27-0975DA379BB5} = {DFB6ADD7-3149-43D9-AFA0-FC4A818B472B}
1114-
{FE23BB65-276A-4E41-8CC7-F7752241DEBA} = {39CDF34B-FB23-49AE-AB27-0975DA379BB5}
11151100
{CBC96CC7-65AB-46EA-A82E-F6A788DABF80} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}
11161101
{E31F9B59-FCF1-4D04-8762-C7BB60285A7B} = {F7876C9B-FB6A-4EFB-B058-D6967DB75FB2}
11171102
EndGlobalSection

‎azure-pipelines.yml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,33 @@ stages:
713713
continueOnError: true
714714
condition: always()
715715

716+
# Build benchmarks
717+
- job: Plain_Build_Benchmarks
718+
pool:
719+
name: $(DncEngPublicBuildPool)
720+
demands: ImageOverride -equals $(WindowsMachineQueueName)
721+
variables:
722+
- name: _BuildConfig
723+
value: Debug
724+
steps:
725+
- checkout: self
726+
clean: true
727+
- script: dotnet --list-sdks
728+
displayName: Report dotnet SDK versions
729+
- task: UseDotNet@2
730+
displayName: install SDK
731+
inputs:
732+
packageType: sdk
733+
useGlobalJson: true
734+
includePreviewVersions: true
735+
workingDirectory: $(Build.SourcesDirectory)
736+
installationPath: $(Agent.ToolsDirectory)/dotnet
737+
- script: dotnet build .\FSharp.Benchmarks.sln /bl:\"artifacts/log/$(_BuildConfig)/BenchmarkBuild.binlog\"
738+
workingDirectory: $(Build.SourcesDirectory)
739+
displayName: Regular rebuild of FSharp.Benchmarks.sln
740+
continueOnError: true
741+
condition: always()
742+
716743
# Test trimming on Windows
717744
- job: Build_And_Test_Trimming_Windows
718745
pool:

‎buildtools/AssemblyCheck/SkipVerifyEmbeddedPdb.txt

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
FSharp.Build.UnitTests.dll
22
FSharp.Compiler.Benchmarks.dll
3-
Fsharp.ProfilingStartpointProject.dll
43
FSharp.Compiler.ComponentTests.dll
54
FSharp.Test.Utilities.dll
65
FSharp.Compiler.Private.Scripting.UnitTests.dll

‎tests/benchmarks/CompiledCodeBenchmarks/MicroPerf/CS/MicroPerfCSharp.csproj

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,11 +6,10 @@
66
<LangVersion>8.0</LangVersion>
77
</PropertyGroup>
88

9-
<ItemGroup>
10-
<Compile Remove="obj\**" />
11-
<EmbeddedResource Remove="obj\**" />
12-
<None Remove="obj\**" />
13-
</ItemGroup>
9+
<PropertyGroup>
10+
<!--missing XML comments-->
11+
<NoWarn>$(NoWarn);CS1591</NoWarn>
12+
</PropertyGroup>
1413

1514
<ItemGroup>
1615
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />

‎tests/benchmarks/CompiledCodeBenchmarks/TaskPerf/TaskPerfCSharp/TaskPerfCSharp.csproj

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,11 @@
66
<LangVersion>8.0</LangVersion>
77
</PropertyGroup>
88

9+
<PropertyGroup>
10+
<!--missing XML comments-->
11+
<NoWarn>$(NoWarn);CS1591</NoWarn>
12+
</PropertyGroup>
13+
914
<ItemGroup>
1015
<PackageReference Include="BenchmarkDotNet" Version="$(BenchmarkDotNetVersion)" />
1116
<PackageReference Include="System.Interactive.Async" Version="4.1.1" />

‎tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/DecentlySizedStandAloneFileBenchmark.fs

Lines changed: 52 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,22 +1,61 @@
11
namespace FSharp.Compiler.Benchmarks
22

33
open System.IO
4-
open HistoricalBenchmark
4+
open FSharp.Compiler.CodeAnalysis
5+
open FSharp.Compiler.Text
56
open BenchmarkDotNet.Attributes
67

7-
type SingleFileCompilerWithILCacheClearing(file, options) =
8-
inherit SingleFileCompiler(file, options)
9-
10-
override this.Cleanup() =
11-
base.Cleanup()
12-
FSharp.Compiler.AbstractIL.ILBinaryReader.ClearAllILModuleReaderCache()
8+
type private SingleFileCompilerConfig =
9+
{
10+
Checker : FSharpChecker
11+
Options : FSharpProjectOptions
12+
}
1313

1414
[<MemoryDiagnoser>]
1515
type DecentlySizedStandAloneFileBenchmark() =
16-
inherit SingleFileCompilerBenchmarkBase(
17-
SingleFileCompilerWithILCacheClearing(
18-
Path.Combine(__SOURCE_DIRECTORY__, "../decentlySizedStandAloneFile.fs"),
19-
OptionsCreationMethod.FromScript
20-
)
21-
)
2216

17+
let mutable configOpt : SingleFileCompilerConfig option = None
18+
let filePath = Path.Combine(__SOURCE_DIRECTORY__, "../decentlySizedStandAloneFile.fs")
19+
20+
let getFileSourceText (filePath : string) =
21+
let text = File.ReadAllText(filePath)
22+
SourceText.ofString text
23+
24+
let getConfig () =
25+
configOpt
26+
|> Option.defaultWith (fun () -> failwith "Setup not run")
27+
28+
[<GlobalSetup>]
29+
member _.Setup() =
30+
configOpt <-
31+
match configOpt with
32+
| Some _ -> configOpt
33+
| None ->
34+
let checker = FSharpChecker.Create(projectCacheSize = 200)
35+
let options =
36+
checker.GetProjectOptionsFromScript(filePath, getFileSourceText filePath)
37+
|> Async.RunSynchronously
38+
|> fst
39+
{
40+
Checker = checker
41+
Options = options
42+
}
43+
|> Some
44+
45+
[<Benchmark>]
46+
member _.Run() =
47+
let config = getConfig()
48+
let _, result =
49+
config.Checker.ParseAndCheckFileInProject(filePath, 0, getFileSourceText filePath, config.Options)
50+
|> Async.RunSynchronously
51+
52+
match result with
53+
| FSharpCheckFileAnswer.Aborted -> failwith "checker aborted"
54+
| FSharpCheckFileAnswer.Succeeded results ->
55+
if results.Diagnostics.Length > 0 then failwithf $"had errors: %A{results.Diagnostics}"
56+
57+
[<IterationCleanup>]
58+
member _.Cleanup() =
59+
let checker = getConfig().Checker
60+
checker.InvalidateAll()
61+
checker.ClearLanguageServiceRootCachesAndCollectAndFinalizeAllTransients()

‎tests/benchmarks/FCSBenchmarks/CompilerServiceBenchmarks/FSharp.Compiler.Benchmarks.fsproj

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
<ItemGroup>
2525
<ProjectReference Include="..\..\..\..\src\Compiler\FSharp.Compiler.Service.fsproj" />
2626
<ProjectReference Include="..\..\..\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj" />
27-
<ProjectReference Include="..\BenchmarkComparison\HistoricalBenchmark.fsproj" />
2827
</ItemGroup>
2928

3029
<ItemGroup Condition="'$(FSHARPCORE_USE_PACKAGE)' != 'true'">

‎tests/benchmarks/Fsharp.ProfilingStartpointProject/Fsharp.ProfilingStartpointProject.fsproj

Lines changed: 0 additions & 16 deletions
This file was deleted.

0 commit comments

Comments
 (0)