treat missing dynamic data as inconclusive

src/Adapter/MSTest.TestAdapter/Discovery/AssemblyEnumerator.cs

@@ -298,7 +298,19 @@ private static bool ProcessTestDataSourceTests(UnitTestElement test, MethodInfo
     {
         foreach (FrameworkITestDataSource dataSource in testDataSources)
         {
-            IEnumerable<object?[]> data = dataSource.GetData(methodInfo);
+            IEnumerable<object?[]>? data = null;
+            try
+            {
+                data = dataSource.GetData(methodInfo);
+            }
+            catch (Exception ex) when (ex is ArgumentException && MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive)
+            {
+                var discoveredTest = test.Clone();
+                discoveredTest.DisplayName = dataSource.GetDisplayName(methodInfo, null) ?? discoveredTest.DisplayName;
+                tests.Add(discoveredTest);
+                continue;
+            }
+
             var testDisplayNameFirstSeen = new Dictionary<string, int>();
             var discoveredTests = new List<UnitTestElement>();
             int index = 0;

src/Adapter/MSTest.TestAdapter/Execution/TestMethodRunner.cs

@@ -1,4 +1,4 @@
-// Copyright (c) Microsoft Corporation. All rights reserved.
+// Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System.Diagnostics;
@@ -294,7 +294,22 @@ private bool ExecuteDataSourceBasedTests(List<TestResult> results)
                 foreach (UTF.ITestDataSource testDataSource in testDataSources)
                 {
                     isDataDriven = true;
-                    foreach (object?[] data in testDataSource.GetData(_testMethodInfo.MethodInfo))
+                    IEnumerable<object?[]>? dataSource = null;
+                    try
+                    {
+                        dataSource = testDataSource.GetData(_testMethodInfo.MethodInfo);
+                    }
+                    catch (Exception ex) when (ex is ArgumentException && MSTestSettings.CurrentSettings.ConsiderEmptyDataSourceAsInconclusive)
+                    {
+                        var inconclusiveResult = new TestResult
+                        {
+                            Outcome = UTF.UnitTestOutcome.Inconclusive,
+                        };
+                        results.Add(inconclusiveResult);
+                        continue;
+                    }
+
+                    foreach (object?[] data in dataSource)
                     {
                         try
                         {

src/Adapter/MSTest.TestAdapter/MSTestSettings.cs

@@ -56,6 +56,7 @@ public MSTestSettings()
         ForcedLegacyMode = false;
         TestSettingsFile = null;
         DisableParallelization = false;
+        ConsiderEmptyDataSourceAsInconclusive = false;
         TestTimeout = 0;
         AssemblyInitializeTimeout = 0;
         ClassInitializeTimeout = 0;
@@ -160,6 +161,11 @@ public static RunConfigurationSettings RunConfigurationSettings
     /// </summary>
     internal int AssemblyCleanupTimeout { get; private set; }
 
+    /// <summary>
+    ///  Gets a value indicating whether to enable marking tests with missing dynamic data as Inconclusive.
+    /// </summary>
+    internal bool ConsiderEmptyDataSourceAsInconclusive { get; private set; }
+
     /// <summary>
     ///  Gets specified global ClassInitializeTimeout timeout.
     /// </summary>
@@ -211,6 +217,7 @@ public static void PopulateSettings(MSTestSettings settings)
         CurrentSettings.TreatClassAndAssemblyCleanupWarningsAsErrors = settings.TreatClassAndAssemblyCleanupWarningsAsErrors;
         CurrentSettings.AssemblyInitializeTimeout = settings.AssemblyInitializeTimeout;
         CurrentSettings.AssemblyCleanupTimeout = settings.AssemblyCleanupTimeout;
+        CurrentSettings.ConsiderEmptyDataSourceAsInconclusive = settings.ConsiderEmptyDataSourceAsInconclusive;
         CurrentSettings.ClassInitializeTimeout = settings.ClassInitializeTimeout;
         CurrentSettings.ClassCleanupTimeout = settings.ClassCleanupTimeout;
         CurrentSettings.TestInitializeTimeout = settings.TestInitializeTimeout;
@@ -474,6 +481,16 @@ private static MSTestSettings ToSettings(XmlReader reader)
                             break;
                         }
 
+                    case "CONSIDEREMPTYDATASOURCEASINCONCLUSIVE":
+                        {
+                            if (bool.TryParse(reader.ReadInnerXml(), out bool considerEmptyDataSourceAsInconclusive))
+                            {
+                                settings.ConsiderEmptyDataSourceAsInconclusive = considerEmptyDataSourceAsInconclusive;
+                            }
+
+                            break;
+                        }
+
                     case "ASSEMBLYINITIALIZETIMEOUT":
                         {
                             if (int.TryParse(reader.ReadInnerXml(), out int assemblyInitializeTimeout) && assemblyInitializeTimeout > 0)

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/DynamicDataTests.cs

@@ -0,0 +1,147 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using Microsoft.Testing.Platform.Acceptance.IntegrationTests;
+using Microsoft.Testing.Platform.Acceptance.IntegrationTests.Helpers;
+using Microsoft.Testing.Platform.Helpers;
+
+namespace MSTest.Acceptance.IntegrationTests;
+
+[TestGroup]
+public class DynamicDataTests : AcceptanceTestBase
+{
+    private readonly TestAssetFixture _testAssetFixture;
+    private const string AssetName = "DynamicData";
+
+    // There's a bug in TAFX where we need to use it at least one time somewhere to use it inside the fixture self (AcceptanceFixture).
+    public DynamicDataTests(ITestExecutionContext testExecutionContext, TestAssetFixture testAssetFixture,
+        AcceptanceFixture globalFixture)
+        : base(testExecutionContext)
+    {
+        _testAssetFixture = testAssetFixture;
+    }
+
+    [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))]
+    public async Task SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusive_Passes(string currentTfm)
+    {
+        string runSettings = $"""
+<?xml version="1.0" encoding="utf-8" ?>
+<RunSettings>
+    <RunConfiguration>
+    </RunConfiguration>
+    <MSTest>
+        <ConsiderEmptyDataSourceAsInconclusive>true</ConsiderEmptyDataSourceAsInconclusive>
+    </MSTest>
+</RunSettings>
+""";
+        var testHost = TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, currentTfm);
+
+        string runSettingsFilePath = Path.Combine(testHost.DirectoryName, $"{Guid.NewGuid():N}.runsettings");
+        File.WriteAllText(runSettingsFilePath, runSettings);
+
+        var testHostResult = await testHost.ExecuteAsync($"--settings {runSettingsFilePath}");
+
+        testHostResult.AssertExitCodeIs(ExitCodes.Success);
+
+        testHostResult.AssertOutputContains("skipped Test");
+    }
+
+    [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))]
+    public async Task SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusiveToFalse_Fails(string currentTfm)
+    {
+        string runSettings = $"""
+<?xml version="1.0" encoding="utf-8" ?>
+<RunSettings>
+    <RunConfiguration>
+    </RunConfiguration>
+    <MSTest>
+        <ConsiderEmptyDataSourceAsInconclusive>false</ConsiderEmptyDataSourceAsInconclusive>
+    </MSTest>
+</RunSettings>
+""";
+        var testHost = TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, currentTfm);
+
+        string runSettingsFilePath = Path.Combine(testHost.DirectoryName, $"{Guid.NewGuid():N}.runsettings");
+        File.WriteAllText(runSettingsFilePath, runSettings);
+
+        var testHostResult = await testHost.ExecuteAsync($"--settings {runSettingsFilePath}");
+
+        testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed);
+
+        testHostResult.AssertOutputContains("failed Test");
+    }
+
+    [ArgumentsProvider(nameof(TargetFrameworks.All), typeof(TargetFrameworks))]
+    public async Task SendingEmptyDataToDynamicDataTest_WithoutSettingConsiderEmptyDataSourceAsInconclusive_Fails(string currentTfm)
+    {
+        var testHost = TestHost.LocateFrom(_testAssetFixture.TargetAssetPath, AssetName, currentTfm);
+
+        var testHostResult = await testHost.ExecuteAsync();
+
+        testHostResult.AssertExitCodeIs(ExitCodes.AtLeastOneTestFailed);
+
+        testHostResult.AssertOutputContains("failed Test");
+    }
+
+    [TestFixture(TestFixtureSharingStrategy.PerTestGroup)]
+    public sealed class TestAssetFixture(AcceptanceFixture acceptanceFixture) : TestAssetFixtureBase(acceptanceFixture.NuGetGlobalPackagesFolder)
+    {
+        public string TargetAssetPath => GetAssetPath(AssetName);
+
+        public override IEnumerable<(string ID, string Name, string Code)> GetAssetsToGenerate()
+        {
+            yield return (AssetName, AssetName,
+                SourceCode
+                .PatchTargetFrameworks(TargetFrameworks.All)
+                .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)
+                .PatchCodeWithReplace("$MicrosoftTestingPlatformExtensionsVersion$", MicrosoftTestingPlatformExtensionsVersion)
+                .PatchCodeWithReplace("$MSTestVersion$", MSTestVersion));
+        }
+
+        private const string SourceCode = """
+#file DynamicData.csproj
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <EnableMSTestRunner>true</EnableMSTestRunner>
+    <TargetFrameworks>$TargetFrameworks$</TargetFrameworks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="MSTest.TestAdapter" Version="$MSTestVersion$" />
+    <PackageReference Include="MSTest.TestFramework" Version="$MSTestVersion$" />
+    <PackageReference Include="Microsoft.Testing.Platform" Version="$MicrosoftTestingPlatformVersion$" />
+  </ItemGroup>
+
+</Project>
+
+#file UnitTest1.cs
+
+using System;
+using System.Collections.Generic;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+[TestClass]
+public class TestClass
+{
+    [TestMethod]
+    [DynamicData(nameof(AdditionalData))]
+    [DynamicData(nameof(AdditionalData2))]
+    public void Test()
+    {
+    }
+
+    public static IEnumerable<int> AdditionalData => Array.Empty<int>();
+
+    public static IEnumerable<int> AdditionalData2
+    {
+        get
+        {
+            yield return 2;
+        }
+    }
+}
+""";
+    }
+}

test/IntegrationTests/MSTest.Acceptance.IntegrationTests/testsbaseline.txt

@@ -207,4 +207,16 @@ MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.InfoTests.
 MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.InfoTests.UsingInfoArgument_OutputCorrectVersionForMSTest(string) (net6.0)
 MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.InfoTests.UsingInfoArgument_OutputCorrectVersionForMSTest(string) (net7.0)
 MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.InfoTests.UsingInfoArgument_OutputCorrectVersionForMSTest(string) (net8.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithoutSettingConsiderEmptyDataSourceAsInconclusive_Fails(string) (net462)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithoutSettingConsiderEmptyDataSourceAsInconclusive_Fails(string) (net6.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithoutSettingConsiderEmptyDataSourceAsInconclusive_Fails(string) (net7.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithoutSettingConsiderEmptyDataSourceAsInconclusive_Fails(string) (net8.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusive_Passes(string) (net462)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusive_Passes(string) (net6.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusive_Passes(string) (net7.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusive_Passes(string) (net8.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusiveToFalse_Fails(string) (net462)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusiveToFalse_Fails(string) (net6.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusiveToFalse_Fails(string) (net7.0)
+MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.DynamicDataTests.SendingEmptyDataToDynamicDataTest_WithSettingConsiderEmptyDataSourceAsInconclusiveToFalse_Fails(string) (net8.0)
 MSTest.Acceptance.IntegrationTests.MSTest.Acceptance.IntegrationTests.AssemblyResolverTests.RunningTests_DoesNotHitResourceRecursionIssueAndDoesNotCrashTheRunner()