Run multiple target frameworks and architectures in single vstest.con…

…sole (#3412)

Allow dlls with mixed target frameworks and architectures to run under the same instance of vstest.console.

playground/MSTest1/MSTest1.csproj

@@ -6,9 +6,10 @@
   <Import Project="$(TestPlatformRoot)scripts/build/TestPlatform.Settings.targets" />
 
   <PropertyGroup>
-    <TargetFrameworks Condition=" '$(OS)' == 'WINDOWS_NT' ">$(TargetFrameworks);net472</TargetFrameworks>
+    <TargetFrameworks Condition=" '$(OS)' == 'WINDOWS_NT' ">$(TargetFrameworks);net472;net5.0</TargetFrameworks>
     <!-- We build this on linux as well, and are including ref assemblies for this framework only. -->
     <TargetFrameworks Condition=" '$(OS)' != 'WINDOWS_NT' ">$(TargetFrameworks);net451</TargetFrameworks>
+    <Prefer32Bit>false</Prefer32Bit>
     <IsPackable>false</IsPackable>
   </PropertyGroup>
 

playground/MSTest1/UnitTest1.cs

@@ -13,5 +13,6 @@ public class UnitTest1
     [TestMethod]
     public void TestMethod1()
     {
+        // Thread.Sleep(1000);
     }
 }

playground/TestPlatform.Playground/Program.cs

@@ -39,46 +39,78 @@ static void Main(string[] args)
         var playground = Path.GetFullPath(Path.Combine(here, "..", "..", "..", ".."));
 
         var console = Path.Combine(here, "vstest.console", "vstest.console.exe");
-        var consoleOptions = new ConsoleParameters
-        {
-            LogFilePath = Path.Combine(here, "logs", "log.txt"),
-            TraceLevel = TraceLevel.Verbose,
-        };
 
-        var r = new VsTestConsoleWrapper(console, consoleOptions);
 
         var sourceSettings = @"
                 <RunSettings>
                     <RunConfiguration>
                         <InIsolation>true</InIsolation>
-                        <MaxCpuCount>0</MaxCpuCount>
+                        <MaxCpuCount>4</MaxCpuCount>
                     </RunConfiguration>
                 </RunSettings>
             ";
+
         var sources = new[] {
-            Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll")
+            Path.Combine(playground, "MSTest1", "bin", "Debug", "net472", "MSTest1.dll"),
+            Path.Combine(playground, "MSTest1", "bin", "Debug", "net5.0", "MSTest1.dll"),
+            @"C:\Users\jajares\source\repos\TestProject48\TestProject48\bin\Debug\net48\TestProject48.dll",
+            @"C:\Users\jajares\source\repos\TestProject48\TestProject1\bin\Debug\net48\win10-x64\TestProject1.dll"
         };
 
+        // console mode
+        var settingsFile = Path.GetTempFileName();
+        try
+        {
+            File.WriteAllText(settingsFile, sourceSettings);
+            var process = Process.Start(console, string.Join(" ", sources) + " --settings:" + settingsFile + " --listtests");
+            process.WaitForExit();
+            if (process.ExitCode != 0)
+            {
+                throw new Exception($"Process failed with {process.ExitCode}");
+            }
+        }
+        finally
+        {
+            try { File.Delete(settingsFile); } catch { }
+        }
+
+        // design mode
+        var consoleOptions = new ConsoleParameters
+        {
+            LogFilePath = Path.Combine(here, "logs", "log.txt"),
+            TraceLevel = TraceLevel.Verbose,
+        };
         var options = new TestPlatformOptions();
-        r.RunTestsWithCustomTestHost(sources, sourceSettings, options, new TestRunHandler(), new DebuggerTestHostLauncher());
+        var r = new VsTestConsoleWrapper(console, consoleOptions);
+        var sessionHandler = new TestSessionHandler();
+#pragma warning disable CS0618 // Type or member is obsolete
+        r.StartTestSession(sources, sourceSettings, sessionHandler);
+#pragma warning restore CS0618 // Type or member is obsolete
+        var discoveryHandler = new PlaygroundTestDiscoveryHandler();
+        r.DiscoverTests(sources, sourceSettings, options, sessionHandler.TestSessionInfo, discoveryHandler);
+        r.RunTestsWithCustomTestHost(discoveryHandler.TestCases, sourceSettings, options, sessionHandler.TestSessionInfo, new TestRunHandler(), new DebuggerTestHostLauncher());
     }
 
     public class PlaygroundTestDiscoveryHandler : ITestDiscoveryEventsHandler, ITestDiscoveryEventsHandler2
     {
         private int _testCasesCount;
 
+        public List<TestCase> TestCases { get; internal set; } = new List<TestCase>();
+
         public void HandleDiscoveredTests(IEnumerable<TestCase> discoveredTestCases)
         {
             Console.WriteLine($"[DISCOVERY.PROGRESS]");
             Console.WriteLine(WriteTests(discoveredTestCases));
             _testCasesCount += discoveredTestCases.Count();
+            if (discoveredTestCases != null) { TestCases.AddRange(discoveredTestCases); }
         }
 
         public void HandleDiscoveryComplete(long totalTests, IEnumerable<TestCase> lastChunk, bool isAborted)
         {
             Console.WriteLine($"[DISCOVERY.COMPLETE] aborted? {isAborted}, tests count: {totalTests}");
             Console.WriteLine("Last chunk:");
             Console.WriteLine(WriteTests(lastChunk));
+            if (lastChunk != null) { TestCases.AddRange(lastChunk); }
         }
 
         public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryCompleteEventArgs, IEnumerable<TestCase> lastChunk)
@@ -92,6 +124,7 @@ public void HandleDiscoveryComplete(DiscoveryCompleteEventArgs discoveryComplete
             Console.WriteLine(WriteSources(discoveryCompleteEventArgs.PartiallyDiscoveredSources));
             Console.WriteLine("Not discovered:");
             Console.WriteLine(WriteSources(discoveryCompleteEventArgs.NotDiscoveredSources));
+            if (lastChunk != null) { TestCases.AddRange(lastChunk); }
         }
 
         public void HandleLogMessage(TestMessageLevel level, string message)
@@ -106,7 +139,7 @@ public void HandleRawMessage(string rawMessage)
 
         private static string WriteTests(IEnumerable<TestCase> testCases)
             => testCases?.Any() == true
-                ? "\t" + string.Join("\n\t", testCases.Select(r => r.DisplayName))
+                ? "\t" + string.Join("\n\t", testCases.Select(r => r.Source + " " + r.DisplayName))
                 : "\t<empty>";
 
         private static string WriteSources(IEnumerable<string> sources)
@@ -183,3 +216,28 @@ public int LaunchTestHost(TestProcessStartInfo defaultTestHostStartInfo, Cancell
         }
     }
 }
+
+internal class TestSessionHandler : ITestSessionEventsHandler
+{
+    public TestSessionInfo TestSessionInfo { get; private set; }
+
+    public void HandleLogMessage(TestMessageLevel level, string message)
+    {
+
+    }
+
+    public void HandleRawMessage(string rawMessage)
+    {
+
+    }
+
+    public void HandleStartTestSessionComplete(StartTestSessionCompleteEventArgs eventArgs)
+    {
+        TestSessionInfo = eventArgs.TestSessionInfo;
+    }
+
+    public void HandleStopTestSessionComplete(StopTestSessionCompleteEventArgs eventArgs)
+    {
+
+    }
+}

playground/TestPlatform.Playground/Properties/launchSettings.json

@@ -5,9 +5,9 @@
       "environmentVariables": {
         "VSTEST_CONNECTION_TIMEOUT": "999",
         "VSTEST_DEBUG_NOBP": "1",
-        "VSTEST_RUNNER_DEBUG_ATTACHVS": "1",
-        "VSTEST_HOST_DEBUG_ATTACHVS": "1",
-        "VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS": "1"
+        "VSTEST_RUNNER_DEBUG_ATTACHVS": "0",
+        "VSTEST_HOST_DEBUG_ATTACHVS": "0",
+        "VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS": "0"
       }
     }
   }

scripts/build/TestPlatform.Dependencies.props

@@ -33,9 +33,9 @@
     Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo.
     These versions need to be "statically" readable because we read this file as xml in our build and tests. -->
     <!-- <MSTestFrameworkLatestVersion></MSTestFrameworkLatestVersion> is not here, because we don't build MSTest locally, so we don't have access to the latest version. -->
-    <MSTestFrameworkLatestPreviewVersion>[2.2.9-preview-20220210-07]</MSTestFrameworkLatestPreviewVersion>
-    <MSTestFrameworkLatestStableVersion>[2.2.8]</MSTestFrameworkLatestStableVersion>
-    <MSTestFrameworkRecentStableVersion>[2.2.7]</MSTestFrameworkRecentStableVersion>
+    <MSTestFrameworkLatestPreviewVersion>[2.2.10-preview-20220414-01]</MSTestFrameworkLatestPreviewVersion>
+    <MSTestFrameworkLatestStableVersion>[2.2.10]</MSTestFrameworkLatestStableVersion>
+    <MSTestFrameworkRecentStableVersion>[2.2.8]</MSTestFrameworkRecentStableVersion>
     <MSTestFrameworkMostDownloadedVersion>[2.1.0]</MSTestFrameworkMostDownloadedVersion>
     <MSTestFrameworkPreviousStableVersion>[2.1.0]</MSTestFrameworkPreviousStableVersion>
     <MSTestFrameworkLegacyStableVersion>[1.4.0]</MSTestFrameworkLegacyStableVersion>
@@ -45,9 +45,9 @@
     See Invoke-TestAssetsBuild in scripts/build.ps1. Exact versions are used to avoid Nuget substituting them by closest match, if we make a typo.
     These versions need to be "statically" readable because we read this file as xml in our build and tests. -->
     <!-- <VSTestConsoleLatestVersion></VSTestConsoleLatestVersion> is not here, NETTestSdkVersion is used instead, because that is the version of the locally built latest package.  -->
-    <VSTestConsoleLatestPreviewVersion>[17.2.0-preview-20220131-20]</VSTestConsoleLatestPreviewVersion>
-    <VSTestConsoleLatestStableVersion>[17.1.0]</VSTestConsoleLatestStableVersion>
-    <VSTestConsoleRecentStableVersion>[17.0.0]</VSTestConsoleRecentStableVersion>
+    <VSTestConsoleLatestPreviewVersion>[17.2.0-preview-20220401-08]</VSTestConsoleLatestPreviewVersion>
+    <VSTestConsoleLatestStableVersion>[17.2.0]</VSTestConsoleLatestStableVersion>
+    <VSTestConsoleRecentStableVersion>[17.1.0]</VSTestConsoleRecentStableVersion>
     <VSTestConsoleMostDownloadedVersion>[16.6.1]</VSTestConsoleMostDownloadedVersion>
     <VSTestConsolePreviousStableVersion>[16.11.0]</VSTestConsolePreviousStableVersion>
     <VSTestConsoleLegacyStableVersion>[15.9.2]</VSTestConsoleLegacyStableVersion>

scripts/build/TestPlatform.targets

@@ -3,7 +3,7 @@
   <PropertyGroup>
     <TestPlatformRoot>$(MSBuildThisFileDirectory)..\..\</TestPlatformRoot>
     <CheckEolTargetFramework>false</CheckEolTargetFramework>
-    <NoWarn>$(NoWarn);CA1416</NoWarn>
+    <NoWarn>$(NoWarn);CA1416;RS0037</NoWarn>
   </PropertyGroup>
 
   <Import Project="$(MSBuildThisFileDirectory)TestPlatform.Localization.targets" />

src/AttachVS/AttachVs.cs

@@ -138,19 +138,12 @@ private static bool AttachVs(Process vs, int pid)
                                 }
                             }
                         }
-                        catch (COMException ex)
+                        // Catch the exception if it is COMException coming directly, or coming from methodInvocation, otherwise just let it be.
+                        catch (Exception ex) when (ex is COMException || (ex is TargetInvocationException tie && tie.InnerException is COMException))
                         {
                             Trace($"ComException: Retrying in 250ms.\n{ex}");
                             Thread.Sleep(250);
                         }
-                        catch (TargetInvocationException ex)
-                        {
-                            if (ex.InnerException is not COMException)
-                                throw;
-
-                            Trace($"ComException: Retrying in 250ms.\n{ex}");
-                            Thread.Sleep(250);
-                        }
                     }
                     Marshal.ReleaseComObject(moniker[0]);
 

src/Microsoft.TestPlatform.Client/TestPlatform.cs

@@ -13,7 +13,6 @@
 using Microsoft.VisualStudio.TestPlatform.Common;
 using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework;
 using Microsoft.VisualStudio.TestPlatform.Common.Hosting;
-using Microsoft.VisualStudio.TestPlatform.Common.Logging;
 using Microsoft.VisualStudio.TestPlatform.Common.Utilities;
 using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel;
@@ -71,34 +70,27 @@ protected internal TestPlatform(
         IFileHelper filehelper,
         ITestRuntimeProviderManager testHostProviderManager)
     {
-        TestEngine = testEngine;
+        _testEngine = testEngine;
         _fileHelper = filehelper;
         _testHostProviderManager = testHostProviderManager;
     }
 
-    /// <summary>
-    /// Gets or sets the test engine instance.
-    /// </summary>
-    private ITestEngine TestEngine { get; set; }
+    private readonly ITestEngine _testEngine;
 
     /// <inheritdoc/>
     public IDiscoveryRequest CreateDiscoveryRequest(
         IRequestData requestData,
         DiscoveryCriteria discoveryCriteria!!,
-        TestPlatformOptions options)
+        TestPlatformOptions options,
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
     {
         PopulateExtensions(discoveryCriteria.RunSettings, discoveryCriteria.Sources);
 
         // Initialize loggers.
-        ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData);
+        ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData);
         loggerManager.Initialize(discoveryCriteria.RunSettings);
 
-        ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(discoveryCriteria.RunSettings);
-        TestPlatform.ThrowExceptionIfTestHostManagerIsNull(testHostManager, discoveryCriteria.RunSettings);
-
-        testHostManager.Initialize(TestSessionMessageLogger.Instance, discoveryCriteria.RunSettings);
-
-        IProxyDiscoveryManager discoveryManager = TestEngine.GetDiscoveryManager(requestData, testHostManager, discoveryCriteria);
+        IProxyDiscoveryManager discoveryManager = _testEngine.GetDiscoveryManager(requestData, discoveryCriteria, sourceToSourceDetailMap);
         discoveryManager.Initialize(options?.SkipDefaultAdapters ?? false);
 
         return new DiscoveryRequest(requestData, discoveryCriteria, discoveryManager, loggerManager);
@@ -108,32 +100,17 @@ public IDiscoveryRequest CreateDiscoveryRequest(
     public ITestRunRequest CreateTestRunRequest(
         IRequestData requestData,
         TestRunCriteria testRunCriteria!!,
-        TestPlatformOptions options)
+        TestPlatformOptions options,
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
     {
         IEnumerable<string> sources = GetSources(testRunCriteria);
         PopulateExtensions(testRunCriteria.TestRunSettings, sources);
 
         // Initialize loggers.
-        ITestLoggerManager loggerManager = TestEngine.GetLoggerManager(requestData);
+        ITestLoggerManager loggerManager = _testEngine.GetLoggerManager(requestData);
         loggerManager.Initialize(testRunCriteria.TestRunSettings);
 
-        // TODO: PERF: this will create a testhost manager, and then it will pass that to GetExecutionManager, where it will
-        // be used only when we will run in-process. If we don't run in process, we will throw away the manager we just
-        // created and let the proxy parallel callbacks to create a new one. This seems to be very easy to move to the GetExecutionManager,
-        // and safe as well, so we create the manager only once.
-        // TODO: Of course TestEngine.GetExecutionManager is public api...
-        ITestRuntimeProvider testHostManager = _testHostProviderManager.GetTestHostManagerByRunConfiguration(testRunCriteria.TestRunSettings);
-        TestPlatform.ThrowExceptionIfTestHostManagerIsNull(testHostManager, testRunCriteria.TestRunSettings);
-
-        testHostManager.Initialize(TestSessionMessageLogger.Instance, testRunCriteria.TestRunSettings);
-
-        // NOTE: The custom launcher should not be set when we have test session info available.
-        if (testRunCriteria.TestHostLauncher != null)
-        {
-            testHostManager.SetCustomLauncher(testRunCriteria.TestHostLauncher);
-        }
-
-        IProxyExecutionManager executionManager = TestEngine.GetExecutionManager(requestData, testHostManager, testRunCriteria);
+        IProxyExecutionManager executionManager = _testEngine.GetExecutionManager(requestData, testRunCriteria, sourceToSourceDetailMap);
         executionManager.Initialize(options?.SkipDefaultAdapters ?? false);
 
         return new TestRunRequest(requestData, testRunCriteria, executionManager, loggerManager);
@@ -143,7 +120,8 @@ public ITestRunRequest CreateTestRunRequest(
     public bool StartTestSession(
         IRequestData requestData,
         StartTestSessionCriteria testSessionCriteria!!,
-        ITestSessionEventsHandler eventsHandler)
+        ITestSessionEventsHandler eventsHandler,
+        Dictionary<string, SourceDetail> sourceToSourceDetailMap)
     {
         RunConfiguration runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testSessionCriteria.RunSettings);
         TestAdapterLoadingStrategy strategy = runConfiguration.TestAdapterLoadingStrategy;
@@ -155,7 +133,7 @@ public bool StartTestSession(
             return false;
         }
 
-        IProxyTestSessionManager testSessionManager = TestEngine.GetTestSessionManager(requestData, testSessionCriteria);
+        IProxyTestSessionManager testSessionManager = _testEngine.GetTestSessionManager(requestData, testSessionCriteria, sourceToSourceDetailMap);
         if (testSessionManager == null)
         {
             // The test session manager is null because the combination of runsettings and
@@ -197,13 +175,13 @@ public void UpdateExtensions(
         IEnumerable<string> pathToAdditionalExtensions,
         bool skipExtensionFilters)
     {
-        TestEngine.GetExtensionManager().UseAdditionalExtensions(pathToAdditionalExtensions, skipExtensionFilters);
+        _testEngine.GetExtensionManager().UseAdditionalExtensions(pathToAdditionalExtensions, skipExtensionFilters);
     }
 
     /// <inheritdoc/>
     public void ClearExtensions()
     {
-        TestEngine.GetExtensionManager().ClearExtensions();
+        _testEngine.GetExtensionManager().ClearExtensions();
     }
 
     private static void ThrowExceptionIfTestHostManagerIsNull(

src/Microsoft.TestPlatform.Common/Hosting/ITestRuntimeProviderManager.cs

@@ -1,12 +1,14 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+using System.Collections.Generic;
+
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host;
 
 namespace Microsoft.VisualStudio.TestPlatform.Common.Hosting;
 
 internal interface ITestRuntimeProviderManager
 {
-    ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration);
+    ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration, List<string> sources);
     ITestRuntimeProvider GetTestHostManagerByUri(string hostUri);
 }

src/Microsoft.TestPlatform.Common/Hosting/TestRunTimeProviderManager.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System;
+using System.Collections.Generic;
 
 using Microsoft.VisualStudio.TestPlatform.Common.Logging;
 using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host;
@@ -44,7 +45,7 @@ public ITestRuntimeProvider GetTestHostManagerByUri(string hostUri)
         return host?.Value;
     }
 
-    public virtual ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration)
+    public virtual ITestRuntimeProvider GetTestHostManagerByRunConfiguration(string runConfiguration, List<string> _)
     {
         foreach (var testExtension in _testHostExtensionManager.TestExtensions)
         {

src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyDiscoveryManager.cs

@@ -10,7 +10,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
 /// <summary>
 /// Interface defining the parallel discovery manager
 /// </summary>
-public interface IParallelProxyDiscoveryManager : IParallelOperationManager, IProxyDiscoveryManager
+public interface IParallelProxyDiscoveryManager : IProxyDiscoveryManager
 {
     /// <summary>
     /// Indicates if user requested an abortion

src/Microsoft.TestPlatform.Common/Interfaces/Engine/ClientProtocol/IParallelProxyExecutionManager.cs

@@ -12,7 +12,7 @@ namespace Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
 /// <summary>
 /// Interface defining the parallel execution manager
 /// </summary>
-public interface IParallelProxyExecutionManager : IParallelOperationManager, IProxyExecutionManager
+public interface IParallelProxyExecutionManager : IProxyExecutionManager
 {
     /// <summary>
     /// Handles Partial Run Complete event coming from a specific concurrent proxy execution manager