Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Test adapter loading strategy #3380

Merged
merged 10 commits into from
Feb 18, 2022
Merged

Test adapter loading strategy #3380

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d4f508d
TestAdapterLoadingStrategy cherry-picked from el/17.1.
Haplois Feb 16, 2022
971b1b4
Merge branch 'main' into dev/haplois/test-adapter-loading-strategy
Haplois Feb 16, 2022
12c12b0
Fixed typos in code.
Haplois Feb 16, 2022
b58979f
Code formatting, and localization typos fixed.
Haplois Feb 16, 2022
b3b59bd
Retrieve strategy if no argument is specified.
Haplois Feb 17, 2022
0fd3c9b
Apply suggestions from code review
Haplois Feb 17, 2022
98d2fe3
PR fixes.
Haplois Feb 17, 2022
04b31b4
PR fixes.
Haplois Feb 17, 2022
28ba1a8
PR fixes.
Haplois Feb 17, 2022
df8092f
PR fixes
Haplois Feb 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion TestPlatform.sln
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AttachVS", "src\AttachVS\At
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "playground", "playground", "{6CE2F530-582B-4695-A209-41065E103426}"
ProjectSection(SolutionItems) = preProject
README.md = README.md
playground\README.md = playground\README.md
EndProjectSection
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestPlatform.Playground", "playground\TestPlatform.Playground\TestPlatform.Playground.csproj", "{545A88D3-1AE2-4D39-9B7C-C691768AD17F}"
Expand Down
262 changes: 174 additions & 88 deletions src/Microsoft.TestPlatform.Client/TestPlatform.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,30 +12,31 @@ namespace Microsoft.VisualStudio.TestPlatform.Client;
using System.Linq;
using System.Reflection;

using Discovery;
using Execution;
using Common;
using Common.ExtensionFramework;
using Common.Hosting;
using Common.Logging;
using Microsoft.VisualStudio.TestPlatform.Client.Discovery;
Haplois marked this conversation as resolved.
Show resolved Hide resolved
using Microsoft.VisualStudio.TestPlatform.Client.Execution;
using Microsoft.VisualStudio.TestPlatform.Common;
using Microsoft.VisualStudio.TestPlatform.Common.ExtensionFramework;
using Microsoft.VisualStudio.TestPlatform.Common.Hosting;
using Microsoft.VisualStudio.TestPlatform.Common.Logging;
using Microsoft.VisualStudio.TestPlatform.Common.Utilities;
using CrossPlatEngine;
using ObjectModel;
using Microsoft.VisualStudio.TestPlatform.CrossPlatEngine;
using Microsoft.VisualStudio.TestPlatform.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Client;
using ObjectModel.Engine;
using ObjectModel.Host;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Engine;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Host;
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
using PlatformAbstractions;
using Utilities.Helpers;
using Utilities.Helpers.Interfaces;
using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers;
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;

using ClientResources = Resources.Resources;
using ClientResources = Microsoft.VisualStudio.TestPlatform.Client.Resources.Resources;

/// <summary>
/// Implementation for TestPlatform.
/// </summary>
internal class TestPlatform : ITestPlatform
{

private readonly TestRuntimeProviderManager _testHostProviderManager;

private readonly IFileHelper _fileHelper;
Expand Down Expand Up @@ -87,16 +88,7 @@ public IDiscoveryRequest CreateDiscoveryRequest(
DiscoveryCriteria discoveryCriteria!!,
TestPlatformOptions options)
{

// Update cache with Extension folder's files.
AddExtensionAssemblies(discoveryCriteria.RunSettings);

// Update extension assemblies from source when design mode is false.
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(discoveryCriteria.RunSettings);
if (!runConfiguration.DesignMode)
{
AddExtensionAssembliesFromSource(discoveryCriteria.Sources);
}
PopulateExtensions(discoveryCriteria.RunSettings, discoveryCriteria.Sources);

// Initialize loggers.
var loggerManager = TestEngine.GetLoggerManager(requestData);
Expand All @@ -119,15 +111,8 @@ public ITestRunRequest CreateTestRunRequest(
TestRunCriteria testRunCriteria!!,
TestPlatformOptions options)
{
AddExtensionAssemblies(testRunCriteria.TestRunSettings);

var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testRunCriteria.TestRunSettings);

// Update extension assemblies from source when design mode is false.
if (!runConfiguration.DesignMode)
{
AddExtensionAssembliesFromSource(testRunCriteria);
}
var sources = GetSources(testRunCriteria);
PopulateExtensions(testRunCriteria.TestRunSettings, sources);

// Initialize loggers.
var loggerManager = TestEngine.GetLoggerManager(requestData);
Expand Down Expand Up @@ -156,9 +141,11 @@ public bool StartTestSession(
StartTestSessionCriteria testSessionCriteria!!,
ITestSessionEventsHandler eventsHandler)
{
AddExtensionAssemblies(testSessionCriteria.RunSettings);

var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(testSessionCriteria.RunSettings);
var strategy = runConfiguration.TestAdapterLoadingStrategy;
Haplois marked this conversation as resolved.
Show resolved Hide resolved

AddExtensionAssemblies(testSessionCriteria.RunSettings, strategy);

if (!runConfiguration.DesignMode)
{
return false;
Expand All @@ -177,6 +164,20 @@ public bool StartTestSession(

return testSessionManager.StartSession(eventsHandler, requestData);
}
private void PopulateExtensions(string runSettings, IEnumerable<string> sources)
Haplois marked this conversation as resolved.
Show resolved Hide resolved
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);
var strategy = runConfiguration.TestAdapterLoadingStrategy;
Haplois marked this conversation as resolved.
Show resolved Hide resolved

// Update cache with Extension folder's files.
AddExtensionAssemblies(runSettings, strategy);

// Update extension assemblies from source when design mode is false.
if (!runConfiguration.DesignMode)
{
AddExtensionAssembliesFromSource(sources, strategy);
}
}

/// <summary>
/// The dispose.
Expand Down Expand Up @@ -211,81 +212,54 @@ private void ThrowExceptionIfTestHostManagerIsNull(
}
}

/// <summary>
/// Updates the test adapter paths provided through run settings to be used by the test
/// service.
/// </summary>
///
/// <param name="runSettings">The run settings.</param>
private void AddExtensionAssemblies(string runSettings)

private void AddExtensionAssemblies(string runSettings, TestAdapterLoadingStrategy adapterLoadingStrategy)
{
IEnumerable<string> customTestAdaptersPaths = RunSettingsUtilities.GetTestAdaptersPaths(runSettings);

if (customTestAdaptersPaths != null)
{
foreach (string customTestAdaptersPath in customTestAdaptersPaths)
{
var adapterPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(customTestAdaptersPath));
if (!Directory.Exists(adapterPath))
{
EqtTrace.Warning($"AdapterPath Not Found: {adapterPath}");
continue;
}
var extensionAssemblies = ExpandTestAdapterPaths(customTestAdaptersPath, _fileHelper, adapterLoadingStrategy);

var extensionAssemblies = new List<string>(
_fileHelper.EnumerateFiles(
adapterPath,
SearchOption.AllDirectories,
TestPlatformConstants.TestAdapterEndsWithPattern,
TestPlatformConstants.TestLoggerEndsWithPattern,
TestPlatformConstants.DataCollectorEndsWithPattern,
TestPlatformConstants.RunTimeEndsWithPattern));

if (extensionAssemblies.Count > 0)
if (extensionAssemblies.Any())
{
UpdateExtensions(extensionAssemblies, skipExtensionFilters: false);
}

}
}
}

/// <summary>
/// Updates the extension assemblies from source directory.
/// Updates the test logger paths from source directory.
/// </summary>
///
/// <param name="testRunCriteria">The test run criteria.</param>
private void AddExtensionAssembliesFromSource(TestRunCriteria testRunCriteria)
/// <param name="sources">The list of sources.</param>
private void AddExtensionAssembliesFromSource(IEnumerable<string> sources, TestAdapterLoadingStrategy strategy)
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
IEnumerable<string> sources = testRunCriteria.Sources;
if (testRunCriteria.HasSpecificTests)
// Skip discovery unless we're using the default behavior, or NextToSource is specified.
if (strategy != TestAdapterLoadingStrategy.Default
&& (strategy & TestAdapterLoadingStrategy.NextToSource) != TestAdapterLoadingStrategy.NextToSource)
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
// If the test execution is with a test filter, group them by sources.
sources = testRunCriteria.Tests.Select(tc => tc.Source).Distinct();
return;
}

AddExtensionAssembliesFromSource(sources);
}

/// <summary>
/// Updates the test logger paths from source directory.
/// </summary>
///
/// <param name="sources">The list of sources.</param>
private void AddExtensionAssembliesFromSource(IEnumerable<string> sources)
{
// Currently we support discovering loggers only from Source directory.
var loggersToUpdate = new List<string>();

foreach (var source in sources)
{
var sourceDirectory = Path.GetDirectoryName(source);
if (!string.IsNullOrEmpty(sourceDirectory)
&& _fileHelper.DirectoryExists(sourceDirectory))
if (!string.IsNullOrEmpty(sourceDirectory) && _fileHelper.DirectoryExists(sourceDirectory))
{
var searchOption = GetSearchOption(strategy, SearchOption.TopDirectoryOnly);

loggersToUpdate.AddRange(
_fileHelper.EnumerateFiles(
sourceDirectory,
SearchOption.TopDirectoryOnly,
searchOption,
TestPlatformConstants.TestLoggerEndsWithPattern));
}
}
Expand All @@ -303,21 +277,133 @@ private void AddExtensionAssembliesFromSource(IEnumerable<string> sources)
/// </summary>
private static void AddExtensionAssembliesFromExtensionDirectory()
{
// This method runs before adapter initialization path, ideally we should replace this mechanism
// this is currently required because we need TestHostProvider to be able to resolve.
Haplois marked this conversation as resolved.
Show resolved Hide resolved
var runSettings = RunSettingsManager.Instance.ActiveRunSettings.SettingsXml;
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);
var strategy = runConfiguration.TestAdapterLoadingStrategy;
Haplois marked this conversation as resolved.
Show resolved Hide resolved

var fileHelper = new FileHelper();
var extensionsFolder = Path.Combine(
Path.GetDirectoryName(
typeof(TestPlatform).GetTypeInfo().Assembly.GetAssemblyLocation()),
"Extensions");
var defaultExtensionPaths = Enumerable.Empty<string>();

// Explicit adapter loading
if ((strategy & TestAdapterLoadingStrategy.Explicit) == TestAdapterLoadingStrategy.Explicit)
{
defaultExtensionPaths = RunSettingsUtilities.GetTestAdaptersPaths(runSettings)
.SelectMany(path => ExpandTestAdapterPaths(path, fileHelper, strategy))
.Union(defaultExtensionPaths);
}

var extensionsFolder = Path.Combine(Path.GetDirectoryName(typeof(TestPlatform).GetTypeInfo().Assembly.GetAssemblyLocation()), "Extensions");
if (fileHelper.DirectoryExists(extensionsFolder))
{
var defaultExtensionPaths = fileHelper.EnumerateFiles(
extensionsFolder,
SearchOption.TopDirectoryOnly,
".dll",
".exe");
// Load default runtime providers
if ((strategy & TestAdapterLoadingStrategy.DefaultRuntimeProviders) == TestAdapterLoadingStrategy.DefaultRuntimeProviders)
{
defaultExtensionPaths = fileHelper
.EnumerateFiles(extensionsFolder, SearchOption.TopDirectoryOnly, TestPlatformConstants.RunTimeEndsWithPattern)
.Union(defaultExtensionPaths);
}

// Default extension loader
if (strategy == TestAdapterLoadingStrategy.Default
|| (strategy & TestAdapterLoadingStrategy.ExtensionsDirectory) == TestAdapterLoadingStrategy.ExtensionsDirectory)
{
defaultExtensionPaths = fileHelper
.EnumerateFiles(extensionsFolder, SearchOption.TopDirectoryOnly, ".dll", ".exe")
.Union(defaultExtensionPaths);
}
}

TestPluginCache.Instance.DefaultExtensionPaths = defaultExtensionPaths.Distinct();
Haplois marked this conversation as resolved.
Show resolved Hide resolved
}

private static SearchOption GetSearchOption(TestAdapterLoadingStrategy strategy, SearchOption defaultStrategyOption) {
if (strategy == TestAdapterLoadingStrategy.Default) {
return defaultStrategyOption;
}
Haplois marked this conversation as resolved.
Show resolved Hide resolved

var searchOption = SearchOption.TopDirectoryOnly;
if ((strategy & TestAdapterLoadingStrategy.Recursive) == TestAdapterLoadingStrategy.Recursive)
{
searchOption = SearchOption.AllDirectories;
}

return searchOption;
Haplois marked this conversation as resolved.
Show resolved Hide resolved
Haplois marked this conversation as resolved.
Show resolved Hide resolved
}

TestPluginCache.Instance.DefaultExtensionPaths = defaultExtensionPaths;
private static IEnumerable<string> ExpandTestAdapterPaths(string path, IFileHelper fileHelper, TestAdapterLoadingStrategy strategy)
{
var adapterPath = Path.GetFullPath(Environment.ExpandEnvironmentVariables(path));

// Default behavior is to only accept directories!
Haplois marked this conversation as resolved.
Show resolved Hide resolved
if (strategy == TestAdapterLoadingStrategy.Default)
{
return ExpandAdaptersWithDefaultStrategy(adapterPath, fileHelper);
}

var adapters = ExpandAdaptersWithExplicitStrategy(adapterPath, fileHelper, strategy);

return adapters.Distinct();
Haplois marked this conversation as resolved.
Show resolved Hide resolved
}

private static IEnumerable<string> ExpandAdaptersWithExplicitStrategy(string path, IFileHelper fileHelper, TestAdapterLoadingStrategy strategy)
Haplois marked this conversation as resolved.
Show resolved Hide resolved
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
if ((strategy & TestAdapterLoadingStrategy.Explicit) != TestAdapterLoadingStrategy.Explicit)
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
return Enumerable.Empty<string>();
}

if (fileHelper.Exists(path))
{
return new[] { path };
}
else if (fileHelper.DirectoryExists(path))
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
var searchOption = GetSearchOption(strategy, SearchOption.TopDirectoryOnly);

var adapterPaths = fileHelper.EnumerateFiles(
path,
searchOption,
TestPlatformConstants.TestAdapterEndsWithPattern,
TestPlatformConstants.TestLoggerEndsWithPattern,
TestPlatformConstants.DataCollectorEndsWithPattern,
TestPlatformConstants.RunTimeEndsWithPattern);

return adapterPaths;
}

EqtTrace.Warning(string.Format(CultureInfo.InvariantCulture, "AdapterPath Not Found: {0}", path));
Haplois marked this conversation as resolved.
Show resolved Hide resolved
Haplois marked this conversation as resolved.
Show resolved Hide resolved
return Enumerable.Empty<string>();
}

private static IEnumerable<string> ExpandAdaptersWithDefaultStrategy(string path, IFileHelper fileHelper)
Haplois marked this conversation as resolved.
Show resolved Hide resolved
{
if (!fileHelper.DirectoryExists(path))
{
EqtTrace.Warning(string.Format("AdapterPath Not Found: {0}", path));
Haplois marked this conversation as resolved.
Show resolved Hide resolved

return Enumerable.Empty<string>();
}

return fileHelper.EnumerateFiles(
path,
SearchOption.AllDirectories,
TestPlatformConstants.TestAdapterEndsWithPattern,
TestPlatformConstants.TestLoggerEndsWithPattern,
TestPlatformConstants.DataCollectorEndsWithPattern,
TestPlatformConstants.RunTimeEndsWithPattern);
}

private static IEnumerable<string> GetSources(TestRunCriteria testRunCriteria)
{
IEnumerable<string> sources = testRunCriteria.Sources;
if (testRunCriteria.HasSpecificTests)
{
// If the test execution is with a test filter, group them by sources.
Haplois marked this conversation as resolved.
Show resolved Hide resolved
sources = testRunCriteria.Tests.Select(tc => tc.Source).Distinct();
}

return sources;
}
Haplois marked this conversation as resolved.
Show resolved Hide resolved
}
1 change: 1 addition & 0 deletions src/Microsoft.TestPlatform.Common/ExtensionFramework/TestExtensionManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,7 @@ public LazyExtension<TExtension, TMetadata> TryGetTestExtension(string extension
break;
}
}

return testExtension;
}

Expand Down
10 changes: 10 additions & 0 deletions src/Microsoft.TestPlatform.Common/Utilities/RunSettingsUtilities.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,4 +190,14 @@ public static IEnumerable<string> GetTestAdaptersPaths(string runSettings)
return testAdaptersPaths;
}

/// <summary>
/// Gets the test adapter loading strategy
/// </summary>
/// <param name="runSettings">Test run settings</param>
/// <returns>Test adapter loading strategy</returns>
internal static TestAdapterLoadingStrategy GetLoadingStrategy(string runSettings) {
Haplois marked this conversation as resolved.
Show resolved Hide resolved
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runSettings);

return runConfiguration?.TestAdapterLoadingStrategy ?? TestAdapterLoadingStrategy.Default;
Haplois marked this conversation as resolved.
Show resolved Hide resolved
Haplois marked this conversation as resolved.
Show resolved Hide resolved
}
}
2 changes: 2 additions & 0 deletions src/Microsoft.TestPlatform.ObjectModel/Friends.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
[assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Extensions.MSPhoneAdapter, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("datacollector, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Common, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("vstest.console, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("Microsoft.VisualStudio.TestPlatform.Client, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]

[assembly: InternalsVisibleTo("Microsoft.TestPlatform.ObjectModel.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
[assembly: InternalsVisibleTo("datacollector.UnitTests, PublicKey=002400000480000094000000060200000024000052534131000400000100010007d1fa57c4aed9f0a32e84aa0faefd0de9e8fd6aec8f87fb03766c834c99921eb23be79ad9d5dcc1dd9ad236132102900b723cf980957fc4e177108fc607774f29e8320e92ea05ece4e821c0a5efe8f1645c4c0c93c1ab99285d622caa652c1dfad63d745d6f2de5f17e5eaf0fc4963d261c8a12436518206dc093344d5ad293")]
Expand Down
Loading