Fix no-suitable provider found (#4475)

Author: nohwnd

src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyDiscoveryManager.cs

@@ -99,7 +99,17 @@ public void DiscoverTests(DiscoveryCriteria discoveryCriteria, ITestDiscoveryEve
         // marked as NotDiscovered.
         _dataAggregator.MarkSourcesWithStatus(discoveryCriteria.Sources, DiscoveryStatus.NotDiscovered);
 
-        _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, InitializeDiscoverTestsOnConcurrentManager, DiscoverTestsOnConcurrentManager);
+        if (nonRunnableWorkloads.Count > 0)
+        {
+            // We found some sources that don't associate to any runtime provider and so they cannot run.
+            // Mark the sources as skipped.
+
+            _dataAggregator.MarkSourcesWithStatus(nonRunnableWorkloads.SelectMany(w => w.Work.Sources), DiscoveryStatus.SkippedDiscovery);
+            // TODO: in strict mode keep them as non-discovered, and mark the run as aborted.
+            // _dataAggregator.MarkAsAborted();
+        }
+
+        _parallelOperationManager.StartWork(runnableWorkloads, eventHandler, GetParallelEventHandler, InitializeDiscoverTestsOnConcurrentManager, DiscoverTestsOnConcurrentManager);
     }
 
     private ITestDiscoveryEventsHandler2 GetParallelEventHandler(ITestDiscoveryEventsHandler2 eventHandler, IProxyDiscoveryManager concurrentManager)

src/Microsoft.TestPlatform.CrossPlatEngine/Client/Parallel/ParallelProxyExecutionManager.cs

@@ -115,7 +115,7 @@ public int StartTestRun(TestRunCriteria testRunCriteria, IInternalTestRunEventsH
             // _currentRunDataAggregator.MarkAsAborted();
         }
 
-        _parallelOperationManager.StartWork(workloads, eventHandler, GetParallelEventHandler, PrepareTestRunOnConcurrentManager, StartTestRunOnConcurrentManager);
+        _parallelOperationManager.StartWork(runnableWorkloads, eventHandler, GetParallelEventHandler, PrepareTestRunOnConcurrentManager, StartTestRunOnConcurrentManager);
 
         // Why 1? Because this is supposed to be a processId, and that is just the default that was chosen by someone before me,
         // and maybe is checked somewhere, but I don't see it checked in our codebase.

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DiscoveryTests.cs

@@ -9,6 +9,7 @@
 
 using Microsoft.TestPlatform.TestUtilities;
 using Microsoft.VisualStudio.TestPlatform.Common.Utilities;
+using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions;
 using Microsoft.VisualStudio.TestTools.UnitTesting;
 
 namespace Microsoft.TestPlatform.AcceptanceTests;
@@ -114,4 +115,28 @@ public void TypesToLoadAttributeTests()
             CollectionAssert.AreEquivalent(expected, actual, $"Specified types using TypesToLoadAttribute in \"{extension}\" assembly doesn't match the expected.");
         }
     }
+
+    [TestMethod]
+    [TestCategory("Windows-Review")]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    public void DiscoverTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(RunnerInfo runnerInfo)
+    {
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var firstAssetDirectory = Path.GetDirectoryName(GetAssetFullPath("MSTestProject1.dll"))!;
+
+        // Include all dlls from the assembly dir, as the default AzDO filter *test*.dll does.
+        var dlls = Directory.EnumerateFiles(firstAssetDirectory, "*test*.dll").ToArray();
+        var quotedDlls = string.Join(" ", dlls.Select(a => a.AddDoubleQuote()));
+
+        var arguments = PrepareArguments(quotedDlls, GetTestAdapterPath(), string.Empty, framework: string.Empty, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path);
+        arguments = string.Concat(arguments, " /listtests");
+        arguments = string.Concat(arguments, " /logger:\"console;prefix=true\"");
+        InvokeVsTest(arguments);
+
+        var portableAssembly = dlls.Single(a => a.EndsWith("Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll"));
+        StringAssert.Contains(StdOut, $"Skipping source: {portableAssembly} (.NETPortable,Version=v4.5,Profile=Profile259,");
+
+        ExitCodeEquals(0);
+    }
 }

test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs

@@ -9,6 +9,7 @@
 using Microsoft.VisualStudio.TestPlatform.CoreUtilities.Extensions;
 
 using TestPlatform.TestUtilities;
+using System.Linq;
 
 namespace Microsoft.TestPlatform.AcceptanceTests;
 
@@ -396,4 +397,27 @@ public void ExitCodeShouldNotDependOnFailTreatNoTestsAsErrorFalseValueWhenThereA
         // Returning 1 because of failing test in test assembly (SimpleTestProject2.dll)
         ExitCodeEquals(1);
     }
+
+    [TestMethod]
+    [TestCategory("Windows-Review")]
+    [NetFullTargetFrameworkDataSource(inIsolation: true, inProcess: true)]
+    public void ExecuteTestsShouldSucceedWhenAtLeastOneDllFindsRuntimeProvider(RunnerInfo runnerInfo)
+    {
+        SetTestEnvironment(_testEnvironment, runnerInfo);
+
+        var firstAssetDirectory = Path.GetDirectoryName(GetAssetFullPath("MSTestProject1.dll"))!;
+
+        // Include all dlls from the assembly dir, as the default AzDO filter *test*.dll does.
+        var dlls = Directory.EnumerateFiles(firstAssetDirectory, "*test*.dll").ToArray();
+        var quotedDlls = string.Join(" ", dlls.Select(a => a.AddDoubleQuote()));
+
+        var arguments = PrepareArguments(quotedDlls, GetTestAdapterPath(), string.Empty, framework: string.Empty, _testEnvironment.InIsolationValue, resultsDirectory: TempDirectory.Path);
+        arguments = string.Concat(arguments, " /logger:\"console;prefix=true\"");
+        InvokeVsTest(arguments);
+
+        var portableAssembly = dlls.Single(a => a.EndsWith("Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface.dll"));
+        StringAssert.Contains(StdOut, $"Skipping source: {portableAssembly} (.NETPortable,Version=v4.5,Profile=Profile259,");
+
+        ExitCodeEquals(1);
+    }
 }

test/Microsoft.TestPlatform.TestUtilities/IntegrationTestBase.cs

@@ -843,6 +843,7 @@ protected static void ExecuteApplication(string path, string? args, out string s
         {
             // Ensure async buffers are flushed
             process.WaitForExit();
+            process.WaitForExit(1000);
         }
 
         stopwatch.Stop();