Parallelize tests

eng/Build.ps1

@@ -362,14 +362,16 @@ function VerifyAssemblyVersionsAndSymbols() {
     }
 }
 
-function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false) {
+function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false, [string] $settings = "") {
     $dotnetPath = InitializeDotNetCli
     $dotnetExe = Join-Path $dotnetPath "dotnet.exe"
     $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject)
-    $testLogPath = "$ArtifactsDir\TestResults\$configuration\${projectName}_$targetFramework.xml"
+    # {assembly} and {framework} will expand respectively. See https://github.com/spekt/testlogger/wiki/Logger-Configuration#logfilepath
+    # This is useful to deconflict log filenames when there are many test assemblies, e.g. when testing a whole solution.
+    $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}.xml"
     $testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog"
     $args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath"
-    $args += " --blame --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=false"
+    $args += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=true"
 
     if (-not $noVisualStudio -or $norestore) {
         $args += " --no-restore"
@@ -379,17 +381,20 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str
         $args += " --no-build"
     }
 
+    $args += " $settings"
+
     if ($asBackgroundJob) {
+        Write-Host
         Write-Host("Starting on the background: $args")
         Write-Host("------------------------------------")
-        $bgJob = Start-Job -ScriptBlock {
-            & $using:dotnetExe test $using:testProject -c $using:configuration -f $using:targetFramework -v n --test-adapter-path $using:testadapterpath --logger "xunit;LogFilePath=$using:testLogPath" /bl:$using:testBinLogPath  --blame --results-directory $using:ArtifactsDir\TestResults\$using:configuration
+        Start-Job -ScriptBlock {
+            $argArray = $using:args -Split " "
+            & $using:dotnetExe $argArray
             if ($LASTEXITCODE -ne 0) {
                 throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args"
             }
         }
-        return $bgJob
-    } else{
+    } else {
         Write-Host("$args")
         Exec-Console $dotnetExe $args
     }
@@ -595,21 +600,20 @@ try {
     $script:BuildCategory = "Test"
     $script:BuildMessage = "Failure running tests"
 
-    if ($testCoreClr) {
-        $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
-
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
-        TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
+    function Receive($job) {
+        while($job.HasMoreData) {
+            Receive-Job $job | Write-Host
+            Start-Sleep -Seconds 1
+        }
+        Receive-Job $job -Wait -ErrorAction Stop
+    }
 
-        # Collect output from  background jobs
-        Wait-job $bgJob | out-null
-        Receive-Job $bgJob -ErrorAction Stop
+    if ($testCoreClr) {
+        $cpuLimit = if ($ci) { "-m:2 -- xUnit.MaxParallelThreads=0.25x" } else { "" }
+        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit
     }
 
-    if ($testDesktop) {
+    if ($testDesktop -and $ci) {
         $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true
 
         TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
@@ -618,9 +622,11 @@ try {
         TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\"
         TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\"
 
-        # Collect output from  background jobs
-        Wait-job $bgJob | out-null
-        Receive-Job $bgJob -ErrorAction Stop
+        Receive -job $bgJob
+    }
+
+    if ($testDesktop -and -not $ci ) {
+        TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\"
     }
 
     if ($testFSharpQA) {

eng/build.sh

@@ -219,7 +219,8 @@ function Test() {
   projectname=$(basename -- "$testproject")
   projectname="${projectname%.*}"
   testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml"
-  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false"
+  args+=" -- xUnit.MaxParallelThreads=3"
   "$DOTNET_INSTALL_DIR/dotnet" $args || exit $?
 }
 

tests/FSharp.Build.UnitTests/FSharp.Build.UnitTests.fsproj

@@ -4,7 +4,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
-    <TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
     <OutputType>Library</OutputType>
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
     <UnitTestType>xunit</UnitTestType>
@@ -18,6 +18,12 @@
     <Compile Include="MapSourceRootsTests.fs" />
   </ItemGroup>
 
+  <ItemGroup>
+    <Content Include="xunit.runner.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content> 
+  </ItemGroup>
+
   <ItemGroup>
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Build\FSharp.Build.fsproj" />
     <ProjectReference Include="$(FSharpSourcesRoot)\FSharp.Core\FSharp.Core.fsproj" />

tests/FSharp.Build.UnitTests/xunit.runner.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeAssembly": true
+}

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/misc/utf8output.fs

@@ -7,6 +7,7 @@ open FSharp.Test
 open FSharp.Test.Compiler
 open System
 
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module utf8output =
 
     [<Fact>]

tests/FSharp.Compiler.ComponentTests/CompilerOptions/fsc/times/times.fs

@@ -8,6 +8,8 @@ open FSharp.Test.Compiler
 open System
 open System.IO
 
+// reportTime uses global state.
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module times =
 
     // This test was automatically generated (moved from FSharpQA suite - CompilerOptions/fsc/times)

tests/FSharp.Compiler.ComponentTests/CompilerService/AsyncMemoize.fs

@@ -28,7 +28,7 @@ let internal observe (cache: AsyncMemoize<_,_,_>) =
 
     cache.Event.Add arrivals.Post
 
-    let next () = collected.Receive(10_000)
+    let next () = collected.Receive()
 
     next
 

tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/CustomAttributes/AttributeUsage/AssemblyVersion04.fs

@@ -14,5 +14,5 @@ let success =
     asm.Version.Major = 1 &&
     asm.Version.Minor = 2 &&
     asm.Version.Build = 3 &&
-    (abs (asm.Version.Revision - (int defaultRevision))) < 10  // default value is seconds in the current day / 2.  Check if within 10 sec of that.
+    (abs (asm.Version.Revision - (int defaultRevision))) < 60  // default value is seconds in the current day / 2.  Check if within 60 sec of that.
 if success then () else failwith "Failed: 1"

tests/FSharp.Compiler.ComponentTests/Conformance/BasicGrammarElements/Events/Basic/Basic.fs

@@ -6,6 +6,7 @@ open Xunit
 open FSharp.Test
 open FSharp.Test.Compiler
 
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module Events =
 
     let verifyCompile compilation =

tests/FSharp.Compiler.ComponentTests/FSharpChecker/TransparentCompiler.fs

@@ -1,4 +1,5 @@
-module FSharpChecker.TransparentCompiler
+[<Xunit.Collection(nameof FSharp.Test.NotThreadSafeResourceCollection)>]
+module FSharpChecker.TransparentCompiler
 
 open System.Collections.Concurrent
 open System.Diagnostics
@@ -991,6 +992,8 @@ type private LoadClosureTestShim(currentFileSystem: IFileSystem) =
                 ?shouldShadowCopy = shouldShadowCopy
             )
 
+// Because it is mutating FileSystem!
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 module TestsMutatingFileSystem =
 
     [<Theory>]

tests/FSharp.Compiler.ComponentTests/Language/BooleanReturningAndReturnTypeDirectedPartialActivePatternTests.fs

@@ -1,5 +1,7 @@
 // Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
 
+// Because of shared fsi session.
+[<FSharp.Test.RunTestCasesInSequence>]
 module Language.BooleanReturningAndReturnTypeDirectedPartialActivePatternTests
 
 open Xunit

tests/FSharp.Compiler.ComponentTests/Language/SequenceExpressions/SequenceExpressionTests.fs

@@ -1,5 +1,7 @@
 // Copyright (c) Microsoft Corporation.  All Rights Reserved.  See License.txt in the project root for license information.
 
+// Run sequentially because of shared fsiSession.
+[<FSharp.Test.RunTestCasesInSequence>]
 module Language.SequenceExpression.SequenceExpressionTests
 
 open FSharp.Test

tests/FSharp.Compiler.ComponentTests/Miscellaneous/FsharpSuiteMigrated.fs

@@ -36,13 +36,15 @@ module ScriptRunner =
             let res = evalScriptFromDiskInSharedSession engine cu
             match res with
             | CompilationResult.Failure _ -> res
-            | CompilationResult.Success s ->
+            | CompilationResult.Success _ ->
                 if capture.OutText |> TestFramework.outputPassed then
                     res
                 else
-                    failwith $"Results looked correct, but 'TEST PASSED OK' was not printed. Result: %A{s}"       
+                    failwith $"Results looked correct, but 'TEST PASSED OK' was not printed."       
 
-        | _ -> failwith $"Compilation unit other than fsharp is not supported, cannot process %A{cu}"
+        | _ ->
+            printfn $"Cannot process %A{cu}"
+            failwith $"Compilation unit other than fsharp is not supported."
 
 /// This test file was created by porting over (slower) FsharpSuite.Tests
 /// In order to minimize human error, the test definitions have been copy-pasted and this adapter provides implementations of the test functions

tests/FSharp.Compiler.ComponentTests/Miscellaneous/MigratedCoreTests.fs

@@ -93,14 +93,17 @@ let ``comprehensions-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehens
 [<Fact>]
 let ``comprehensions-FSI`` () = singleTestBuildAndRun "core/comprehensions" FSI
 
-[<Fact>]
-let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG
+// Cancels default token.
+[<Collection(nameof NotThreadSafeResourceCollection)>]
+module Comprehensionshw =
+    [<Fact>]
+    let ``comprehensionshw-FSC_DEBUG`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_DEBUG
 
-[<Fact>]
-let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED
+    [<Fact>]
+    let ``comprehensionshw-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/comprehensions-hw" FSC_OPTIMIZED
 
-[<Fact>]
-let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI
+    [<Fact>]
+    let ``comprehensionshw-FSI`` () = singleTestBuildAndRun "core/comprehensions-hw" FSI
 
 [<Fact>]
 let ``genericmeasures-FSC_DEBUG`` () = singleTestBuildAndRun "core/genericmeasures" FSC_DEBUG
@@ -375,18 +378,21 @@ let ``recordResolution-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/recordRe
 [<Fact>]
 let ``recordResolution-FSI`` () = singleTestBuildAndRun "core/recordResolution" FSI
 
-// This test has hardcoded expectations about current synchronization context
-// Will be moved out of FsharpSuite.Tests in a later phase for desktop framework
-[<FactForNETCOREAPP>]
-let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED
+// Cancels default token.
+[<Collection(nameof NotThreadSafeResourceCollection)>]
+module CoreControl =
+    // This test has hardcoded expectations about current synchronization context
+    // Will be moved out of FsharpSuite.Tests in a later phase for desktop framework
+    [<FactForNETCOREAPP>]
+    let ``control-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/control" FSC_OPTIMIZED
 
-[<FactForNETCOREAPP>]
-let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI
+    [<FactForNETCOREAPP>]
+    let ``control-FSI`` () = singleTestBuildAndRun "core/control" FSI
 
-[<FactForNETCOREAPP>]
-let ``control --tailcalls`` () =
-    let cfg =  "core/control"
-    singleTestBuildAndRunAux cfg  ["--tailcalls"] FSC_OPTIMIZED
+    [<FactForNETCOREAPP>]
+    let ``control --tailcalls`` () =
+        let cfg =  "core/control"
+        singleTestBuildAndRunAux cfg  ["--tailcalls"] FSC_OPTIMIZED
 
 [<Fact>]
 let ``controlChamenos-FSC_OPTIMIZED`` () =
@@ -401,7 +407,7 @@ let ``controlChamenos-FSI`` () =
 [<Fact>]
 let ``controlMailbox-FSC_OPTIMIZED`` () = singleTestBuildAndRun "core/controlMailbox" FSC_OPTIMIZED
 
-[<Fact(Skip="Flaky")>]
+[<Fact>]
 let ``controlMailbox-FSI`` () = singleTestBuildAndRun "core/controlMailbox" FSI
 
 [<Fact>]

tests/FSharp.Compiler.ComponentTests/TypeChecks/Graph/Utils.fs

@@ -5,7 +5,7 @@ open FSharp.Compiler.GraphChecking
 open FSharp.Compiler.Text
 open FSharp.Compiler.Syntax
 
-let private checker = FSharpChecker.Create()
+open FSharp.Test
 
 let parseSourceCode (name: string, code: string) =
     let sourceText = SourceText.ofString code
@@ -16,7 +16,7 @@ let parseSourceCode (name: string, code: string) =
         }
 
     let result =
-        checker.ParseFile(name, sourceText, parsingOptions) |> Async.RunSynchronously
+        CompilerAssert.Checker.ParseFile(name, sourceText, parsingOptions) |> Async.RunSynchronously
 
     result.ParseTree
 

tests/FSharp.Compiler.ComponentTests/xunit.runner.json

@@ -1,6 +1,5 @@
 {
   "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
   "appDomain": "denied",
-  "parallelizeTestCollections": false,
-  "maxParallelThreads": 1
+  "parallelizeAssembly": true
 }

tests/FSharp.Compiler.Private.Scripting.UnitTests/DependencyManagerInteractiveTests.fs

@@ -26,6 +26,7 @@ module Native =
 
 type scriptHost (?langVersion: LangVersion) = inherit FSharpScript(langVersion=defaultArg langVersion LangVersion.Preview)
 
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 type DependencyManagerInteractiveTests() =
 
     let getValue ((value: Result<FsiValue option, exn>), (errors: FSharpDiagnostic[])) =

tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharp.Compiler.Private.Scripting.UnitTests.fsproj

@@ -3,7 +3,7 @@
 
   <PropertyGroup>
     <TargetFrameworks>net472;$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
-    <TargetFrameworks Condition="'$(OS)' == 'Unix'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
+    <TargetFrameworks Condition="'$(OS)' == 'Unix' or '$(BUILDING_USING_DOTNET)' == 'true'">$(FSharpNetCoreProductTargetFramework)</TargetFrameworks>
     <OutputType>Library</OutputType>
     <DisableImplicitFSharpCoreReference>true</DisableImplicitFSharpCoreReference>
     <UnitTestType>xunit</UnitTestType>

tests/FSharp.Compiler.Private.Scripting.UnitTests/xunit.runner.json

@@ -1,6 +1,5 @@
 {
-    "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
-    "appDomain": "denied",
-    "parallelizeTestCollections": false,
-    "maxParallelThreads": 1
+  "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json",
+  "appDomain": "denied",
+  "parallelizeAssembly": true
 }

tests/FSharp.Compiler.Service.Tests/AssemblyContentProviderTests.fs

@@ -22,7 +22,7 @@ let private projectOptions : FSharpProjectOptions =
       UnresolvedReferences = None
       Stamp = None }
 
-let private checker = FSharpChecker.Create(useTransparentCompiler=CompilerAssertHelpers.UseTransparentCompiler)
+let private checker = FSharpChecker.Create(useTransparentCompiler = CompilerAssertHelpers.UseTransparentCompiler)
 
 let private assertAreEqual (expected, actual) =
     if actual <> expected then

tests/FSharp.Compiler.Service.Tests/Common.fs

@@ -31,7 +31,7 @@ type Async with
         task.Result
 
 // Create one global interactive checker instance
-let checker = FSharpChecker.Create(useTransparentCompiler=FSharp.Compiler.CompilerConfig.FSharpExperimentalFeaturesEnabledAutomatically)
+let checker = FSharpChecker.Create(useTransparentCompiler = FSharp.Test.CompilerAssertHelpers.UseTransparentCompiler)
 
 type TempFile(ext, contents: string) =
     let tmpFile =  Path.ChangeExtension(getTemporaryFileName (), ext)

tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.Tests.fsproj

@@ -19,7 +19,6 @@
     <Content Include="expected-help-output.bsl">
       <CopyToOutputDirectory>Never</CopyToOutputDirectory>
     </Content>
-    <Compile Include="AssemblyInfo.fs" />
     <Compile Include="SurfaceArea.fs" Condition="'$(TargetFrameworkIdentifier)' == '.NETCoreApp'" />
     <Compile Include="..\FSharp.Test.Utilities\XunitSetup.fs">
       <Link>XunitSetup.fs</Link>

tests/FSharp.Compiler.Service.Tests/FileSystemTests.fs

@@ -25,6 +25,8 @@ let file2 = """
 module File2
 let B = File1.A + File1.A"""
 
+// FileSystem is a global shared resource.
+[<Collection(nameof NotThreadSafeResourceCollection)>]
 type internal MyFileSystem() =
     inherit DefaultFileSystem()
         static member FilesCache = dict [(fileName1, file1); (fileName2, file2)]

tests/FSharp.Compiler.Service.Tests/FsiHelpTests.fs

@@ -3,7 +3,6 @@
 open FSharp.Test.Assert
 open Xunit
 
-[<CollectionDefinition("FsiHelpTests", DisableParallelization = false)>]
 module FsiHelpTests =
 
     [<Fact>]