Skip to content

Review simplify compatibility sources, deduplicate tests #15472

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
nohwnd merged 6 commits into from
Mar 10, 2026
Merged

Review simplify compatibility sources, deduplicate tests #15472

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
d5248be
Review how we generate tests, simplify compatibility sources, dedupli…
nohwnd Mar 10, 2026
765adb9
Apply suggestions from code review
nohwnd Mar 10, 2026
dd15348
Update test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Extens…
nohwnd Mar 10, 2026
3e9c61e
Update test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Extens…
nohwnd Mar 10, 2026
af29470
Update test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Extens…
nohwnd Mar 10, 2026
c4b41a0
Add back the throw on non-mstest adapter that copilot forced me to re…
nohwnd Mar 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/AcceptanceTestBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@ public class AcceptanceTestBase : IntegrationTestBase
/// Our current defaults for .NET and .NET Framework.
/// </summary>
public const string DEFAULT_HOST_NETFX_AND_NET = "net462;net8.0";
public const string DEFAULT_HOST_NET = "net8.0";
public const string DEFAULT_RUNNER_NETFX_AND_NET = "net48;net10.0";
public const string DEFAULT_RUNNER_NET = "net10.0";
public const string LATEST_TO_LEGACY = "Latest;LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable";
public const string LATESTPREVIEW_TO_LEGACY = "LatestPreview;LatestStable;RecentStable;MostDownloaded;PreviousStable;LegacyStable";
public const string LATEST = "Latest";
Expand Down
19 changes: 1 addition & 18 deletions test/Microsoft.TestPlatform.Acceptance.IntegrationTests/ExecutionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public class ExecutionTests : AcceptanceTestBase
//TODO: It looks like the first 3 tests would be useful to multiply by all 3 test frameworks, should we make the test even more generic, or duplicate them?
[TestMethod]
[TestCategory("Windows-Review")]
[MSTestCompatibilityDataSource(InProcess = true)]

@nohwnd nohwnd Mar 10, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is now default.

[MSTestCompatibilityDataSource()]
public void RunMultipleTestAssemblies(RunnerInfo runnerInfo)
{
SetTestEnvironment(_testEnvironment, runnerInfo);
Expand All @@ -37,23 +37,6 @@ public void RunMultipleTestAssemblies(RunnerInfo runnerInfo)
StdOutHasNoWarnings();
}

[TestMethod]
[TestCategory("Windows-Review")]
[TestPlatformCompatibilityDataSource]
public void RunTestsFromMultipleMSTestAssemblies(RunnerInfo runnerInfo)

@nohwnd nohwnd Mar 10, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same tests as above, just multiplied to 100+ tests which can test combinations of versions that are already released.

{
SetTestEnvironment(_testEnvironment, runnerInfo);

var assemblyPaths = BuildMultipleAssemblyPath("MSTestProject1.dll", "MSTestProject2.dll");

InvokeVsTestForExecution(assemblyPaths, testAdapterPath: null, FrameworkArgValue, string.Empty);

ValidateSummaryStatus(passed: 2, failed: 2, skipped: 2);
ExitCodeEquals(1); // failing tests
StdErrHasTestRunFailedMessageButNoOtherError();
StdOutHasNoWarnings();
}

[TestMethod]
[TestCategory("Windows-Review")]
[TestHostCompatibilityDataSource]
Expand Down
205 changes: 86 additions & 119 deletions .../Microsoft.TestPlatform.Acceptance.IntegrationTests/Extension/CompatibilityRowsBuilder.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,28 +7,29 @@
using System.Linq;
using System.Xml;

using NuGet.Versioning;

using Microsoft.TestPlatform.TestUtilities;

using NuGet.Versioning;

namespace Microsoft.TestPlatform.AcceptanceTests;

public class CompatibilityRowsBuilder
{
private static XmlDocument? s_depsXml;
private readonly string[] _runnerFrameworks;
private readonly string[] _runnerVersions;
private readonly string[] _hostFrameworks;
private readonly string[] _adapterVersions;
private readonly string[] _adapters;

private readonly string[] _runnerVersions;
private readonly string[] _hostVersions;
private readonly string[] _adapterVersions;

public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET,
string runnerVersions = AcceptanceTestBase.LATEST_TO_LEGACY,
string hostFrameworks = AcceptanceTestBase.DEFAULT_HOST_NETFX_AND_NET,
string hostVersions = AcceptanceTestBase.LATEST_TO_LEGACY,
string adapterVersions = AcceptanceTestBase.LATESTPREVIEW_TO_LEGACY,
string adapters = AcceptanceTestBase.MSTEST)
public CompatibilityRowsBuilder(string runnerVersions,
string runnerFrameworks,
string hostVersions,
string hostFrameworks,
string adapterVersions,
string adapters)
{
_runnerFrameworks = runnerFrameworks.Split(';');
_runnerVersions = runnerVersions.Split(';');
Expand All @@ -39,15 +40,11 @@ public CompatibilityRowsBuilder(string runnerFrameworks = AcceptanceTestBase.DEF
}

/// <summary>
/// Add run for in-process using the selected .NET Framework runners, and and all selected adapters.
/// Add run for in-process using the selected .NET Framework runners, and all selected adapters.
/// </summary>
public bool WithInProcess { get; set; } = true;
public bool WithEveryVersionOfRunner { get; set; } = true;
public bool WithEveryVersionOfHost { get; set; } = true;
public bool WithEveryVersionOfAdapter { get; set; } = true;
public bool WithOlderConfigurations { get; set; } = true;
public bool WithInProcess { get; set; }
// Add runner from VSIX to check the shipment we make into VisualStudio.
public bool WithVSIXRunner { get; set; } = true;
public bool WithVSIXRunner { get; set; }

public string? BeforeRunnerFeature { get; set; }
public string? AfterRunnerFeature { get; set; }
Expand All @@ -66,23 +63,13 @@ public List<RunnerInfo> CreateData()
{
var dataRows = new List<RunnerInfo>();

if (WithEveryVersionOfRunner)

@nohwnd nohwnd Mar 10, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There were two sources of the data, idea was that TestPlatformCompatibilitySource, has some more powerful api to generate exactly what is needed, but this makes it really hard to understand what it will do, and that source is for experimentation only anyway. So now we simply generate all combinations of the input data (with some special handling for in process - just because we don't have "unknown" for testhost version.

All in all this is not needed anymore, and was causing bugs for mstest source, which provided all test runner versions, and WithEveryVersionOfRunner = false, which had no effect. So tests were generated for every runner version, which does not make sense.

AddEveryVersionOfRunner(dataRows);

if (WithEveryVersionOfHost)
AddEveryVersionOfHost(dataRows);

if (WithEveryVersionOfAdapter)
AddEveryVersionOfAdapter(dataRows);

if (WithOlderConfigurations)
AddOlderConfigurations(dataRows);
AddRows(dataRows);

if (WithInProcess)
AddInProcess(dataRows);

if (WithVSIXRunner)
AddVsix(dataRows);
AddVsix(dataRows, WithInProcess);

var minVersion = ParseAndPatchSemanticVersion("0.0.0-alpha.1");
var maxVersion = ParseAndPatchSemanticVersion("9999.0.0");
Expand Down Expand Up @@ -145,9 +132,44 @@ public List<RunnerInfo> CreateData()
&& r.TestHostInfo != null && isInRange(ParseAndPatchSemanticVersion(r.TestHostInfo.Version), beforeTestHostVersion, afterTestHostVersion)
&& r.AdapterInfo != null && isInRange(ParseAndPatchSemanticVersion(r.AdapterInfo.Version), beforeAdapterVersion, afterAdapterVersion)).ToList();

// We use ToString to determine which values are unique. Not great solution, but works better than using records.
// Figure out the distinct rows, and shrink the values, so if we have multiple rows that use the same versions and same setup, and only differ in how the version types
// are called (e.g. Latest and LatestStable have the same version), then we get just single row, with description for both.
// like RunMultipleTestAssemblies1 (Row: 0, Matrix, Runner = net10.0, TargetFramework = net462, InIsolation, vstest.console = 18.6.0-dev [Latest], Testhost = 18.6.0-dev [Latest], MSTest = 3.9.3 [LatestPreview, LatestStable])
var distinctRows = new Dictionary<string, RunnerInfo>();
rows.ForEach(r => distinctRows[r.ToString()] = r);
foreach (var r in rows)
{
var key = $"{r.Batch}|{r.RunnerFramework}|{r.VSTestConsoleInfo!.Version}|{r.TargetFramework}|{r.TestHostInfo!.Version}|{r.InIsolationValue}|{r.AdapterInfo!.Name}|{r.AdapterInfo.Version}";
Comment thread
nohwnd marked this conversation as resolved.
if (distinctRows.TryGetValue(key, out var value))
{
if (value.VSTestConsoleInfo!.Version == r.VSTestConsoleInfo.Version)
{
value.VSTestConsoleInfo.VersionType += $",{r.VSTestConsoleInfo.VersionType}";
}

if (value.TestHostInfo!.Version == r.TestHostInfo.Version)
{
value.TestHostInfo.VersionType += $",{r.TestHostInfo.VersionType}";
}

if (r.AdapterInfo!.Version == value.AdapterInfo!.Version)
{
value.AdapterInfo.VersionType += $",{r.AdapterInfo.VersionType}";
}
}
else
{
distinctRows.Add(key, r);
}
}

// We added all the version types together (would have been better to have this property as array originally, but that would complicate other code), now deduplicate the names to make it less confusing for user.
// Latest, Latest, Latest, LatestPreview -> Latest, LatestPreview
foreach (var r in distinctRows.Values)
{
r.VSTestConsoleInfo!.VersionType = string.Join(", ", r.VSTestConsoleInfo!.VersionType!.Split(',').Distinct());
r.TestHostInfo!.VersionType = string.Join(", ", r.TestHostInfo!.VersionType!.Split(',').Distinct());
r.AdapterInfo!.VersionType = string.Join(", ", r.AdapterInfo!.VersionType!.Split(',').Distinct());
Comment thread
nohwnd marked this conversation as resolved.
}

if (distinctRows.Count == 0)
{
Expand All @@ -172,133 +194,78 @@ private static SemanticVersion ParseAndPatchSemanticVersion(string? version)
return SemanticVersion.Parse(v?.TrimStart('v'));
}

private void AddInProcess(List<RunnerInfo> dataRows)
private void AddRows(List<RunnerInfo> dataRows)
{
foreach (var runnerFramework in _runnerFrameworks)
{
if (!runnerFramework.StartsWith("net4"))
{
continue;
}

foreach (var runnerVersion in _runnerVersions)
{
foreach (var adapter in _adapters)
{
foreach (var adapterVersion in _adapterVersions)
{
AddRow(dataRows, "In process", runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapterVersion, inIsolation: false);
}
}
}
}
}

private void AddVsix(List<RunnerInfo> dataRows)
{
foreach (var hostFramework in _hostFrameworks)
{
AddRow(dataRows, "VSIX", AcceptanceTestBase.LATESTVSIX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.LATEST, hostFramework, AcceptanceTestBase.LATESTSTABLE, inIsolation: true);
AddRow(dataRows, "VSIX", AcceptanceTestBase.LATESTVSIX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.LATEST, hostFramework, AcceptanceTestBase.LATESTSTABLE, inIsolation: false);
}
}

private void AddOlderConfigurations(List<RunnerInfo> dataRows)
{
// Older configurations where the runner, host and adapter version are the same.
// We already added the row where all are newest when adding combination with all runners.
foreach (var runnerVersion in _runnerVersions)

@nohwnd nohwnd Mar 10, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This whole method tries to be less clever. Whatever you specify, it will give you all combinations. Before we had different methods that tried to encode some of the thinking (e.g. when testing testhost, we don't need every version of runner), now instead this is done via the input data - you provide just 1 version of the runner to use, and so you will get 1 combination.

{
foreach (var runnerFramework in _runnerFrameworks)
{
foreach (var hostFramework in _hostFrameworks)
{
var isNetFramework = hostFramework.StartsWith("net4");
var hostVersion = runnerVersion;
foreach (var _ in _adapters)
// .NET Framework testhost ships with the runner, and the version from the
// runner directory is always selected, otherwise select the newest version from _hostFrameworks.
Comment thread
nohwnd marked this conversation as resolved.
var hostVersions = isNetFramework ? [runnerVersion] : _hostVersions;
foreach (var hostVersion in hostVersions)
{
var adapterVersion = _adapterVersions[0];
AddRow(dataRows, "Older", runnerVersion, runnerFramework, hostVersion, hostFramework, adapterVersion, inIsolation: true);
foreach (var adapterVersion in _adapterVersions)
{
foreach (var adapter in _adapters)
{
AddRow(dataRows, "Matrix", runnerVersion, runnerFramework, hostVersion, hostFramework, adapterVersion, adapter, inIsolation: true);
}
}
}
}
}
}
}

private void AddEveryVersionOfAdapter(List<RunnerInfo> dataRows)
private void AddInProcess(List<RunnerInfo> dataRows)
{
var runnerVersion = _runnerVersions[0];
foreach (var runnerFramework in _runnerFrameworks)
{
foreach (var hostFramework in _hostFrameworks)
if (!runnerFramework.StartsWith("net4"))
{
continue;
}

foreach (var runnerVersion in _runnerVersions)
{
var isNetFramework = hostFramework.StartsWith("net4");
// .NET Framework testhost ships with the runner, and the version from the
// runner directory is always selected, otherwise select the newest version from _hostFrameworks.
var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0];
foreach (var adapter in _adapters)
{
foreach (var adapterVersion in _adapterVersions)
{
AddRow(dataRows, "Every adapter", runnerVersion, runnerFramework, hostVersion, hostFramework, adapterVersion, inIsolation: true);
AddRow(dataRows, "In process", runnerVersion, runnerFramework, runnerVersion, runnerFramework, adapterVersion, adapter, inIsolation: false);
}
}
}
}
}

private void AddEveryVersionOfHost(List<RunnerInfo> dataRows)
private void AddVsix(List<RunnerInfo> dataRows, bool inProcess)
{
var runnerVersion = _runnerVersions[0];

foreach (var runnerFramework in _runnerFrameworks)
// Runs outside of process, test both tfms of testhost.
foreach (var hostFramework in _hostFrameworks)
{
foreach (var hostFramework in _hostFrameworks)
{
var isNetFramework = hostFramework.StartsWith("net4");
// .NET Framework testhost ships with the runner, and the version from the
// runner directory is always the same as the runner. There are no variations
// so we just need to add host versions for .NET testhosts.
var hostVersions = isNetFramework ? [] : _hostVersions.ToArray();
foreach (var hostVersion in hostVersions)
{
foreach (var _ in _adapters)
{
// use the newest
var adapterVersion = _adapterVersions[0];
AddRow(dataRows, "Every host", runnerVersion, runnerFramework, hostVersion, hostFramework, adapterVersion, inIsolation: true);
}
}
}
AddRow(dataRows, "VSIX", AcceptanceTestBase.LATESTVSIX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.LATEST, hostFramework, AcceptanceTestBase.LATESTSTABLE, AcceptanceTestBase.MSTEST, inIsolation: true);
}
}

private void AddEveryVersionOfRunner(List<RunnerInfo> dataRows)
{
foreach (var runnerVersion in _runnerVersions)
if (inProcess)
{
foreach (var runnerFramework in _runnerFrameworks)
{
foreach (var hostFramework in _hostFrameworks)
{
var isNetFramework = hostFramework.StartsWith("net4");
// .NET Framework testhost ships with the runner, and the version from the
// runner directory is always selected, otherwise select the newest version from _hostFrameworks.
var hostVersion = isNetFramework ? runnerVersion : _hostVersions[0];
foreach (var _ in _adapters)
{
// use the newest
var adapterVersion = _adapterVersions[0];
AddRow(dataRows, "Every runner", runnerVersion, runnerFramework, hostVersion, hostFramework, adapterVersion, inIsolation: true);
}
}
}
// Runs in process. We specify the testhost, but it has no impact.
AddRow(dataRows, "VSIX", AcceptanceTestBase.LATESTVSIX, AcceptanceTestBase.DEFAULT_RUNNER_NETFX, AcceptanceTestBase.LATEST, AcceptanceTestBase.DEFAULT_HOST_NETFX, AcceptanceTestBase.LATESTSTABLE, AcceptanceTestBase.MSTEST, inIsolation: false);
}
Comment thread
nohwnd marked this conversation as resolved.
}

private void AddRow(List<RunnerInfo> dataRows, string batch,
string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapterVersion, bool inIsolation)
string runnerVersion, string runnerFramework, string hostVersion, string hostFramework, string adapterVersion, string adapter, bool inIsolation)
{
if (adapter != AcceptanceTestBase.MSTEST)
{
throw new NotSupportedException($"Adapter {adapter} is not supported. Only {AcceptanceTestBase.MSTEST} is supported.");
}

RunnerInfo runnerInfo = GetRunnerInfo(batch, runnerFramework, hostFramework, inIsolation);
runnerInfo.DebugInfo = GetDebugInfo();
runnerInfo.VSTestConsoleInfo = GetVSTestConsoleInfo(runnerVersion, runnerInfo);
Expand Down
Loading
Loading