Run multiple target frameworks and architectures in single vstest.console

scripts/build.ps1

@@ -123,6 +123,23 @@ if ($env:PATH -notlike "*$attachVsPath") {
     $env:PATH = "$attachVsPath;$env:PATH"
 }
 
+# VsixUtil gets regularly eaten by antivirus or something. Remove the package dir if it gets broken
+# so nuget restores it correctly.
+$vsSdkBuildToolsVersion = ([xml](Get-Content $env:TP_ROOT_DIR\scripts\build\TestPlatform.Dependencies.props)).Project.PropertyGroup.VSSdkBuildToolsVersion
+$vsixUtilDir = "$env:TP_ROOT_DIR\packages\microsoft.vssdk.buildtools"
+if ((Test-Path $vsixUtilDir) -and -not (Test-Path "$vsixUtilDir\$vsSdkBuildToolsVersion\tools\vssdk\bin\VsixUtil.exe"))
+{
+    Remove-Item -Recurse -Force $vsixUtilDir
+}
+
+# Procdump gets regularly eaten by antivirus or something. Remove the package dir if it gets broken
+# so nuget restores it correctly.
+$procdumpDir = "$env:TP_ROOT_DIR\packages\procdump"
+if ((Test-Path $procdumpDir) -and 2 -ne @(Get-Item "$procdumpDir\0.0.1\bin").Length)
+{
+    Remove-Item -Recurse -Force $procdumpDir
+}
+
 function Invoke-Build
 {
     $timer = Start-Timer

test/DataCollectors/Microsoft.TestPlatform.Extensions.EventLogCollector.UnitTests/EventLogDataCollectorTests.cs

@@ -217,11 +217,10 @@ public void InitializeShouldSubscribeToDataCollectionEvents()
     {
         var testableDataCollectionEvents = new TestableDataCollectionEvents();
         _eventLogDataCollector.Initialize(null, testableDataCollectionEvents, _mockDataCollectionSink, _mockDataCollectionLogger.Object, _dataCollectionEnvironmentContext);
-        Assert.AreEqual(1, testableDataCollectionEvents.GetTestHostLaunchedInvocationList().Length);
-        Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList().Length);
-        Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList().Length);
-        Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList().Length);
-        Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList().Length);
+        Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseStartInvocationList()?.Length, "GetTestCaseStartInvocationList");
+        Assert.AreEqual(1, testableDataCollectionEvents.GetTestCaseEndInvocationList()?.Length, "GetTestCaseEndInvocationList");
+        Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionEndInvocationList()?.Length, "GetTestSessionEndInvocationList");
+        Assert.AreEqual(1, testableDataCollectionEvents.GetTestSessionStartInvocationList()?.Length, "GetTestSessionStartInvocationList");
     }
 
     [TestMethod]

test/Microsoft.TestPlatform.AcceptanceTests/AppDomainTests.cs

@@ -29,22 +29,21 @@ public void RunTestExecutionWithDisableAppDomain(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        using var tempDir = new TempDirectory();
-        var testAppDomainDetailFileName = Path.Combine(tempDir.Path, "appdomain_test.txt");
-        var dataCollectorAppDomainDetailFileName = Path.Combine(tempDir.Path, "appdomain_datacollector.txt");
+        var testAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_test.txt");
+        var dataCollectorAppDomainDetailFileName = Path.Combine(TempDirectory.Path, "appdomain_datacollector.txt");
 
         // Delete test output files if already exist
         File.Delete(testAppDomainDetailFileName);
         File.Delete(dataCollectorAppDomainDetailFileName);
 
-        var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, tempDir);
+        var runsettingsFilePath = GetInProcDataCollectionRunsettingsFile(true, TempDirectory);
         var arguments = PrepareArguments(
             GetSampleTestAssembly(),
             GetTestAdapterPath(),
             runsettingsFilePath,
             FrameworkArgValue,
             runnerInfo.InIsolationValue,
-            tempDir.Path);
+            TempDirectory.Path);
 
         // Sets the environment variables used by the test project and test data collector.
         var env = new Dictionary<string, string>

test/Microsoft.TestPlatform.AcceptanceTests/ArgumentProcessorTests.cs

@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-using Microsoft.TestPlatform.TestUtilities;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 #nullable disable
@@ -41,8 +40,7 @@ public void PassingInvalidArgumentsToVsTestConsoleShouldNotPrintHelpMessage(Runn
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        using var tempDir = new TempDirectory();
-        var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path);
+        var arguments = PrepareArguments(GetSampleTestAssembly(), GetTestAdapterPath(), string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path);
         arguments = string.Concat(arguments, " /badArgument");
 
         InvokeVsTest(arguments);

test/Microsoft.TestPlatform.AcceptanceTests/BlameDataCollectorTests.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Xml;
 
 using Microsoft.TestPlatform.TestUtilities;
@@ -23,6 +24,19 @@ public class BlameDataCollectorTests : AcceptanceTestBase
 {
     public const string NETCOREANDFX = "net452;net472;netcoreapp3.1";
     public const string NET50 = "net5.0";
+    private readonly string _procDumpPath;
+
+    public BlameDataCollectorTests()
+    {
+        _procDumpPath = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin");
+        var procDumpExePath = Path.Combine(_procDumpPath, "procdump.exe");
+        if (!File.Exists(procDumpExePath))
+        {
+            throw new InvalidOperationException($"Procdump path {procDumpExePath} does not exist. "
+                + "It is possible that antivirus deleted it from your nuget cache. "
+                + "Delete the whole procdump folder in your nuget cache, and run build, or restore");
+        }
+    }
 
     [TestMethod]
     // netcoreapp2.1 dump is not supported on Linux
@@ -31,15 +45,14 @@ public class BlameDataCollectorTests : AcceptanceTestBase
     [NetCoreTargetFrameworkDataSource]
     public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInfo)
     {
-        using var tempDir = new TempDirectory();
         SetTestEnvironment(_testEnvironment, runnerInfo);
         var assemblyPaths = GetAssetFullPath("BlameUnitTestProject.dll");
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, FrameworkArgValue, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame");
-        arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}");
+        arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");
         InvokeVsTest(arguments);
 
-        VaildateOutput(tempDir, "BlameUnitTestProject.UnitTest1.TestMethod2");
+        VaildateOutput(TempDirectory, "BlameUnitTestProject.UnitTest1.TestMethod2");
     }
 
     [TestMethod]
@@ -49,23 +62,22 @@ public void BlameDataCollectorShouldGiveCorrectTestCaseName(RunnerInfo runnerInf
     [NetCoreTargetFrameworkDataSource]
     public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo)
     {
-        using var tempDir = new TempDirectory();
 
         SetTestEnvironment(_testEnvironment, runnerInfo);
         var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject3.dll").Trim('\"');
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame:CollectDump");
-        arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}");
+        arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");
         arguments = string.Concat(arguments, " /testcasefilter:ExitWithStackoverFlow");
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath,
         };
 
         InvokeVsTest(arguments, env);
 
-        VaildateOutput(tempDir, "SampleUnitTestProject3.UnitTest1.ExitWithStackoverFlow", validateDumpFile: true);
+        VaildateOutput(TempDirectory, "SampleUnitTestProject3.UnitTest1.ExitWithStackoverFlow", validateDumpFile: true);
     }
 
     [TestMethod]
@@ -75,18 +87,17 @@ public void BlameDataCollectorShouldOutputDumpFile(RunnerInfo runnerInfo)
     [NetCoreTargetFrameworkDataSource]
     public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInfo runnerInfo)
     {
-        using var tempDir = new TempDirectory();
 
         SetTestEnvironment(_testEnvironment, runnerInfo);
         var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"');
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame:CollectDump");
-        arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}");
+        arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");
         arguments = string.Concat(arguments, " /testcasefilter:PassingTest");
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath
         };
 
         InvokeVsTest(arguments, env);
@@ -101,23 +112,22 @@ public void BlameDataCollectorShouldNotOutputDumpFileWhenNoCrashOccurs(RunnerInf
     [NetCoreTargetFrameworkDataSource]
     public void BlameDataCollectorShouldOutputDumpFileWhenNoCrashOccursButCollectAlwaysIsEnabled(RunnerInfo runnerInfo)
     {
-        using var tempDir = new TempDirectory();
 
         SetTestEnvironment(_testEnvironment, runnerInfo);
         var assemblyPaths = BuildMultipleAssemblyPath("SimpleTestProject.dll").Trim('\"');
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty, string.Empty, runnerInfo.InIsolationValue);
         arguments = string.Concat(arguments, $" /Blame:CollectDump;CollectAlways=True");
-        arguments = string.Concat(arguments, $" /ResultsDirectory:{tempDir.Path}");
+        arguments = string.Concat(arguments, $" /ResultsDirectory:{TempDirectory.Path}");
         arguments = string.Concat(arguments, " /testcasefilter:PassingTest");
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath
         };
 
         InvokeVsTest(arguments, env);
 
-        Assert.IsTrue(StdOut.Contains(".dmp"), "it should collect dump, even if nothing crashed");
+        StringAssert.Matches(StdOut, new Regex("\\.dmp"), "it should collect dump, even if nothing crashed");
     }
 
     [TestMethod]
@@ -133,7 +143,7 @@ public void HangDumpOnTimeout(RunnerInfo runnerInfo)
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath
         };
 
         InvokeVsTest(arguments, env);
@@ -156,7 +166,7 @@ public void CrashDumpWhenThereIsNoTimeout(RunnerInfo runnerInfo)
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath
         };
 
         InvokeVsTest(arguments, env);
@@ -179,7 +189,7 @@ public void CrashDumpOnExit(RunnerInfo runnerInfo)
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath
         };
 
         InvokeVsTest(arguments, env);
@@ -200,7 +210,7 @@ public void CrashDumpOnStackOverflow(RunnerInfo runnerInfo)
 
         var env = new Dictionary<string, string>
         {
-            ["PROCDUMP_PATH"] = Path.Combine(_testEnvironment.PackageDirectory, @"procdump\0.0.1\bin"),
+            ["PROCDUMP_PATH"] = _procDumpPath
         };
 
         InvokeVsTest(arguments, env);
@@ -313,14 +323,14 @@ private void ValidateDump(int expectedDumpCount = 1)
         }
     }
 
-    private void VaildateOutput(TempDirectory tempDir, string testName, bool validateDumpFile = false)
+    private void VaildateOutput(TempDirectory tempDirectory, string testName, bool validateDumpFile = false)
     {
         bool isSequenceAttachmentReceived = false;
         bool isDumpAttachmentReceived = false;
         bool isValid = false;
         StdErrorContains(testName);
         StdOutputContains("Sequence_");
-        var resultFiles = Directory.GetFiles(tempDir.Path, "*", SearchOption.AllDirectories);
+        var resultFiles = Directory.GetFiles(tempDirectory.Path, "*", SearchOption.AllDirectories);
 
         foreach (var file in resultFiles)
         {
@@ -335,9 +345,9 @@ private void VaildateOutput(TempDirectory tempDir, string testName, bool validat
             }
         }
 
-        Assert.IsTrue(isSequenceAttachmentReceived);
-        Assert.IsTrue(!validateDumpFile || isDumpAttachmentReceived);
-        Assert.IsTrue(isValid);
+        Assert.IsTrue(isSequenceAttachmentReceived, "Sequence attachment was not received.");
+        Assert.IsTrue(!validateDumpFile || isDumpAttachmentReceived, "Dump attachment was not received.");
+        Assert.IsTrue(isValid, "Sequence attachment is not valid.");
     }
 
     private bool IsValidXml(string xmlFilePath)

test/Microsoft.TestPlatform.AcceptanceTests/CUITTest.cs

@@ -18,20 +18,19 @@ public class CuitTest : AcceptanceTestBase
     public void CuitRunAllTests(RunnerInfo runnerInfo)
     {
         SetTestEnvironment(_testEnvironment, runnerInfo);
-        CuitRunAll(runnerInfo.RunnerFramework);
+        CuitRunAll(runnerInfo);
     }
 
-    private void CuitRunAll(string runnerFramework)
+    private void CuitRunAll(RunnerInfo runnerInfo)
     {
-        if (runnerFramework.StartsWith("netcoreapp"))
+        if (runnerInfo.IsNetRunner)
         {
-            Assert.Inconclusive("CUIT tests are not supported with .Netcore runner.");
+            Assert.Inconclusive("CUIT tests are not supported with .NET Core runner.");
             return;
         }
 
         var assemblyAbsolutePath = _testEnvironment.GetTestAsset("CUITTestProject.dll", "net451");
-        using var tempDir = new TempDirectory();
-        var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: tempDir.Path);
+        var arguments = PrepareArguments(assemblyAbsolutePath, string.Empty, string.Empty, FrameworkArgValue, resultsDirectory: TempDirectory.Path);
 
         InvokeVsTest(arguments);
         ValidateSummaryStatus(1, 0, 0);

test/Microsoft.TestPlatform.AcceptanceTests/CodeCoverageTests.cs

@@ -208,10 +208,9 @@ public void CollectCodeCoverageSpecifyOutputFormatCoberturaOverrideRunSettingsCo
 
     private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParameters)
     {
-        using var tempDir = new TempDirectory();
         SetTestEnvironment(_testEnvironment, runnerInfo);
 
-        var arguments = CreateArguments(tempDir, runnerInfo, testParameters, out var trxFilePath);
+        var arguments = CreateArguments(TempDirectory, runnerInfo, testParameters, out var trxFilePath);
 
         InvokeVsTest(arguments);
 
@@ -220,8 +219,8 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam
             testParameters.ExpectedSkippedTests,
             testParameters.ExpectedFailedTests);
 
-        var actualCoverageFile = GetCoverageFileNameFromTrx(trxFilePath, tempDir.Path);
-        Console.WriteLine($@"Coverage file: {actualCoverageFile}  Results directory: {tempDir.Path} trxfile: {trxFilePath}");
+        var actualCoverageFile = GetCoverageFileNameFromTrx(trxFilePath, TempDirectory.Path);
+        Console.WriteLine($@"Coverage file: {actualCoverageFile}  Results directory: {TempDirectory.Path} trxfile: {trxFilePath}");
         Assert.IsTrue(File.Exists(actualCoverageFile), "Coverage file not found: {0}", actualCoverageFile);
 
         if (testParameters.RunSettingsType == TestParameters.SettingsType.XmlOutput)
@@ -237,7 +236,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam
             Assert.IsTrue(actualCoverageFile.EndsWith(".coverage", StringComparison.InvariantCultureIgnoreCase));
         }
 
-        var coverageDocument = GetXmlCoverage(actualCoverageFile, tempDir);
+        var coverageDocument = GetXmlCoverage(actualCoverageFile, TempDirectory);
         if (testParameters.CheckSkipped)
         {
             AssertSkippedMethod(coverageDocument);
@@ -247,7 +246,7 @@ private void CollectCodeCoverage(RunnerInfo runnerInfo, TestParameters testParam
     }
 
     private string CreateArguments(
-        TempDirectory tempDir,
+        TempDirectory tempDirectory,
         RunnerInfo runnerInfo,
         TestParameters testParameters,
         out string trxFilePath)
@@ -257,14 +256,14 @@ private string CreateArguments(
         string traceDataCollectorDir = Path.Combine(IntegrationTestEnvironment.TestPlatformRootDirectory,
             "artifacts", IntegrationTestEnvironment.BuildConfiguration, "Microsoft.CodeCoverage");
 
-        string diagFileName = Path.Combine(tempDir.Path, "diaglog.txt");
+        string diagFileName = Path.Combine(tempDirectory.Path, "diaglog.txt");
         var arguments = PrepareArguments(assemblyPaths, GetTestAdapterPath(), string.Empty,
-            FrameworkArgValue, runnerInfo.InIsolationValue, tempDir.Path);
+            FrameworkArgValue, runnerInfo.InIsolationValue, tempDirectory.Path);
         arguments = string.Concat(arguments, $" /Diag:{diagFileName}",
             $" /TestAdapterPath:{traceDataCollectorDir}");
         arguments = string.Concat(arguments, $" /Platform:{testParameters.TargetPlatform}");
 
-        trxFilePath = Path.Combine(tempDir.Path, Guid.NewGuid() + ".trx");
+        trxFilePath = Path.Combine(tempDirectory.Path, Guid.NewGuid() + ".trx");
         arguments = string.Concat(arguments, " /logger:trx;logfilename=" + trxFilePath);
 
         var defaultRunSettingsPath = Path.Combine(