Skip to content

Commit 1f9905a

Browse files
nohwndnohwnd
authored
Forward standard output of testhost (#4998)
* Capture and forward output * Enable passing info messages from design mode client * Add test for output * Add project into solution * Unignore test
1 parent 1133bec commit 1f9905a

File tree

25 files changed

+196
-49
lines changed

25 files changed

+196
-49
lines changed

‎playground/TestPlatform.Playground/Environment.cs‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ internal class EnvironmentVariables
1414
["VSTEST_RUNNER_DEBUG_ATTACHVS"] = "0",
1515
["VSTEST_HOST_DEBUG_ATTACHVS"] = "0",
1616
["VSTEST_DATACOLLECTOR_DEBUG_ATTACHVS"] = "0",
17-
["VSTEST_EXPERIMENTAL_FORWARD_OUTPUT_FEATURE"] = "0"
1817
};
1918

2019
}

‎src/Microsoft.TestPlatform.Client/DesignMode/DesignModeClient.cs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Logging;
2626
using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
2727
using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions.Interfaces;
28-
using Microsoft.VisualStudio.TestPlatform.Utilities.Helpers.Interfaces;
28+
using Microsoft.VisualStudio.TestPlatform.Utilities;
2929

3030
using CommunicationUtilitiesResources = Microsoft.VisualStudio.TestPlatform.CommunicationUtilities.Resources.Resources;
3131

@@ -43,9 +43,8 @@ public class DesignModeClient : IDesignModeClient
4343
private readonly ProtocolConfig _protocolConfig = Constants.DefaultProtocolConfig;
4444
private readonly IEnvironment _platformEnvironment;
4545
private readonly TestSessionMessageLogger _testSessionMessageLogger;
46+
private readonly bool _forwardOutput;
4647
private readonly object _lockObject = new();
47-
private readonly bool _isForwardingOutput;
48-
4948
[SuppressMessage("Style", "IDE1006:Naming Styles", Justification = "Part of the public API.")]
5049
[SuppressMessage("Design", "CA1051:Do not declare visible instance fields", Justification = "Part of the public API")]
5150
protected Action<Message>? onCustomTestHostLaunchAckReceived;
@@ -57,7 +56,7 @@ public class DesignModeClient : IDesignModeClient
5756
/// Initializes a new instance of the <see cref="DesignModeClient"/> class.
5857
/// </summary>
5958
public DesignModeClient()
60-
: this(new SocketCommunicationManager(), JsonDataSerializer.Instance, new PlatformEnvironment(), new EnvironmentVariableHelper())
59+
: this(new SocketCommunicationManager(), JsonDataSerializer.Instance, new PlatformEnvironment())
6160
{
6261
}
6362

@@ -73,14 +72,14 @@ public DesignModeClient()
7372
/// <param name="platformEnvironment">
7473
/// The platform Environment
7574
/// </param>
76-
internal DesignModeClient(ICommunicationManager communicationManager, IDataSerializer dataSerializer, IEnvironment platformEnvironment, IEnvironmentVariableHelper environmentVariableHelper)
75+
internal DesignModeClient(ICommunicationManager communicationManager, IDataSerializer dataSerializer, IEnvironment platformEnvironment)
7776
{
7877
_communicationManager = communicationManager;
7978
_dataSerializer = dataSerializer;
8079
_platformEnvironment = platformEnvironment;
8180
_testSessionMessageLogger = TestSessionMessageLogger.Instance;
81+
_forwardOutput = !FeatureFlag.Instance.IsSet(FeatureFlag.VSTEST_DISABLE_STANDARD_OUTPUT_FORWARDING);
8282
_testSessionMessageLogger.TestRunMessage += TestRunMessageHandler;
83-
_isForwardingOutput = environmentVariableHelper.GetEnvironmentVariable("VSTEST_EXPERIMENTAL_FORWARD_OUTPUT_FEATURE") == "1";
8483
}
8584

8685
/// <summary>
@@ -449,8 +448,15 @@ public void TestRunMessageHandler(object? sender, TestRunMessageEventArgs e)
449448
case TestMessageLevel.Informational:
450449
EqtTrace.Info(e.Message);
451450

452-
if (_isForwardingOutput || EqtTrace.IsInfoEnabled)
451+
// When forwarding output we need to allow Info messages that originate from this process (or more precisely from the IMessageLogger that we
452+
// are observing, but we don't have an easy way to tell apart "output" and non-output info messages. So when output forwarding is enabled
453+
// we forward any informational message. This is okay, because in worst case we will send few more messages forward that are not output.
454+
// And that is fine, because any adapter has access to SendMessage(MessageLevel.Informational,...) which we don't filter anywhere (it is passed
455+
// as a raw message and forwarded to VS), so any adapter can spam the Tests output in VS as it wants.
456+
if (_forwardOutput || EqtTrace.IsInfoEnabled)
457+
{
453458
SendTestMessage(e.Level, e.Message);
459+
}
454460
break;
455461

456462

‎src/Microsoft.TestPlatform.Common/ExtensionDecorators/ExtensionDecoratorFactory.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ public ExtensionDecoratorFactory(IFeatureFlag featureFlag)
1616

1717
public ITestExecutor Decorate(ITestExecutor originalTestExecutor)
1818
{
19-
return _featureFlag.IsSet(FeatureFlag.DISABLE_SERIALTESTRUN_DECORATOR)
19+
return _featureFlag.IsSet(FeatureFlag.VSTEST_DISABLE_SERIALTESTRUN_DECORATOR)
2020
? originalTestExecutor
2121
: new SerialTestRunDecorator(originalTestExecutor);
2222
}

‎src/Microsoft.TestPlatform.CommunicationUtilities/JsonDataSerializer.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ public class JsonDataSerializer : IDataSerializer
2222
{
2323
private static JsonDataSerializer? s_instance;
2424

25-
private static readonly bool DisableFastJson = FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_FASTER_JSON_SERIALIZATION);
25+
private static readonly bool DisableFastJson = FeatureFlag.Instance.IsSet(FeatureFlag.VSTEST_DISABLE_FASTER_JSON_SERIALIZATION);
2626

2727
private static readonly JsonSerializer PayloadSerializerV1; // payload serializer for version <= 1
2828
private static readonly JsonSerializer PayloadSerializerV2; // payload serializer for version >= 2

‎src/Microsoft.TestPlatform.CoreUtilities/FeatureFlag/FeatureFlag.cs‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,33 +32,42 @@ private FeatureFlag() { }
3232
// Only check the env variable once, when it is not set or is set to 0, consider it unset. When it is anything else, consider it set.
3333
public bool IsSet(string featureFlag) => _cache.GetOrAdd(featureFlag, f => (Environment.GetEnvironmentVariable(f)?.Trim() ?? "0") != "0");
3434

35-
private const string VSTEST_ = nameof(VSTEST_);
36-
3735
// Added for artifact post-processing, it enable/disable the post processing.
3836
// Added in 17.2-preview 7.0-preview
39-
public const string DISABLE_ARTIFACTS_POSTPROCESSING = VSTEST_ + nameof(DISABLE_ARTIFACTS_POSTPROCESSING);
37+
public const string VSTEST_DISABLE_ARTIFACTS_POSTPROCESSING = nameof(VSTEST_DISABLE_ARTIFACTS_POSTPROCESSING);
4038

4139
// Added for artifact post-processing, it will show old output for dotnet sdk scenario.
4240
// It can be useful if we need to restore old UX in case users are parsing the console output.
4341
// Added in 17.2-preview 7.0-preview
44-
public const string DISABLE_ARTIFACTS_POSTPROCESSING_NEW_SDK_UX = VSTEST_ + nameof(DISABLE_ARTIFACTS_POSTPROCESSING_NEW_SDK_UX);
42+
public const string VSTEST_DISABLE_ARTIFACTS_POSTPROCESSING_NEW_SDK_UX = nameof(VSTEST_DISABLE_ARTIFACTS_POSTPROCESSING_NEW_SDK_UX);
4543

4644
// Faster JSON serialization relies on less internals of NewtonsoftJson, and on some additional caching.
47-
public const string DISABLE_FASTER_JSON_SERIALIZATION = VSTEST_ + nameof(DISABLE_FASTER_JSON_SERIALIZATION);
45+
public const string VSTEST_DISABLE_FASTER_JSON_SERIALIZATION = nameof(VSTEST_DISABLE_FASTER_JSON_SERIALIZATION);
4846

4947
// Forces vstest.console to run all sources using the same target framework (TFM) and architecture, instead of allowing
5048
// multiple different tfms and architectures to run at the same time.
51-
public const string DISABLE_MULTI_TFM_RUN = VSTEST_ + nameof(DISABLE_MULTI_TFM_RUN);
49+
public const string VSTEST_DISABLE_MULTI_TFM_RUN = nameof(VSTEST_DISABLE_MULTI_TFM_RUN);
5250

5351
// Disables setting a higher value for SetMinThreads. Setting SetMinThreads value to higher allows testhost to connect back faster
5452
// even though we are blocking additional threads because we don't have to wait for ThreadPool to start more threads.
55-
public const string DISABLE_THREADPOOL_SIZE_INCREASE = VSTEST_ + nameof(DISABLE_THREADPOOL_SIZE_INCREASE);
53+
public const string VSTEST_DISABLE_THREADPOOL_SIZE_INCREASE = nameof(VSTEST_DISABLE_THREADPOOL_SIZE_INCREASE);
5654

5755
// Disable the SerialTestRunDecorator
58-
public const string DISABLE_SERIALTESTRUN_DECORATOR = VSTEST_ + nameof(DISABLE_SERIALTESTRUN_DECORATOR);
56+
public const string VSTEST_DISABLE_SERIALTESTRUN_DECORATOR = nameof(VSTEST_DISABLE_SERIALTESTRUN_DECORATOR);
5957

6058
// Disable setting UTF8 encoding in console.
61-
public const string DISABLE_UTF8_CONSOLE_ENCODING = VSTEST_ + nameof(DISABLE_UTF8_CONSOLE_ENCODING);
59+
public const string VSTEST_DISABLE_UTF8_CONSOLE_ENCODING = nameof(VSTEST_DISABLE_UTF8_CONSOLE_ENCODING);
60+
61+
// VSTEST_EXPERIMENTAL_FORWARD_OUTPUT_FEATURE=1 replaced by the CAPTURING and FORWARDING flags, and was enabling
62+
// the same behavior as what is now the default (both capture and forward set to TRUE).
63+
// Because this is the new default we don't have to handle it in any special way. Setting it to 0 was not defined
64+
// and so it also does not need any special treatment.
65+
//
66+
// Disable capturing standard output of testhost.
67+
public const string VSTEST_DISABLE_STANDARD_OUTPUT_CAPTURING = nameof(VSTEST_DISABLE_STANDARD_OUTPUT_CAPTURING);
68+
69+
// Disable forwarding standard output of testhost.
70+
public const string VSTEST_DISABLE_STANDARD_OUTPUT_FORWARDING = nameof(VSTEST_DISABLE_STANDARD_OUTPUT_FORWARDING);
6271

6372
[Obsolete("Only use this in tests.")]
6473
internal static void Reset()

‎src/Microsoft.TestPlatform.CrossPlatEngine/PostProcessing/ArtifactProcessingManager.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ public void CollectArtifacts(TestRunCompleteEventArgs testRunCompleteEventArgs,
8181
ValidateArg.NotNull(testRunCompleteEventArgs, nameof(testRunCompleteEventArgs));
8282
ValidateArg.NotNull(runSettingsXml, nameof(runSettingsXml));
8383

84-
if (_featureFlag.IsSet(FeatureFlag.DISABLE_ARTIFACTS_POSTPROCESSING))
84+
if (_featureFlag.IsSet(FeatureFlag.VSTEST_DISABLE_ARTIFACTS_POSTPROCESSING))
8585
{
8686
EqtTrace.Verbose("ArtifactProcessingManager.CollectArtifacts: Feature disabled");
8787
return;
@@ -120,7 +120,7 @@ public void CollectArtifacts(TestRunCompleteEventArgs testRunCompleteEventArgs,
120120

121121
public async Task PostProcessArtifactsAsync()
122122
{
123-
if (_featureFlag.IsSet(FeatureFlag.DISABLE_ARTIFACTS_POSTPROCESSING))
123+
if (_featureFlag.IsSet(FeatureFlag.VSTEST_DISABLE_ARTIFACTS_POSTPROCESSING))
124124
{
125125
EqtTrace.Verbose("ArtifactProcessingManager.PostProcessArtifacts: Feature disabled");
126126
return;

‎src/Microsoft.TestPlatform.ObjectModel/PublicAPI/PublicAPI.Shipped.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -972,3 +972,5 @@ virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.ProtectedGetP
972972
virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.TestObject.ProtectedSetPropertyValue(Microsoft.VisualStudio.TestPlatform.ObjectModel.TestProperty! property, object? value) -> void
973973
virtual Microsoft.VisualStudio.TestPlatform.ObjectModel.ValidateValueCallback.Invoke(object? value) -> bool
974974
Microsoft.VisualStudio.TestPlatform.ObjectModel.Architecture.RiscV64 = 8 -> Microsoft.VisualStudio.TestPlatform.ObjectModel.Architecture
975+
Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration.CaptureStandardOutput.get -> bool
976+
Microsoft.VisualStudio.TestPlatform.ObjectModel.RunConfiguration.ForwardStandardOutput.get -> bool

‎src/Microsoft.TestPlatform.ObjectModel/RunSettings/RunConfiguration.cs‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
using Microsoft.VisualStudio.TestPlatform.CoreUtilities;
1010
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Utilities;
1111
using Microsoft.VisualStudio.TestPlatform.PlatformAbstractions;
12+
using Microsoft.VisualStudio.TestPlatform.Utilities;
1213

1314
namespace Microsoft.VisualStudio.TestPlatform.ObjectModel;
1415

@@ -92,6 +93,8 @@ public RunConfiguration() : base(Constants.RunConfigurationSettingsName)
9293
_shouldCollectSourceInformation = false;
9394
TargetDevice = null;
9495
ExecutionThreadApartmentState = Constants.DefaultExecutionThreadApartmentState;
96+
CaptureStandardOutput = !FeatureFlag.Instance.IsSet(FeatureFlag.VSTEST_DISABLE_STANDARD_OUTPUT_CAPTURING);
97+
ForwardStandardOutput = !FeatureFlag.Instance.IsSet(FeatureFlag.VSTEST_DISABLE_STANDARD_OUTPUT_FORWARDING);
9598
}
9699

97100
/// <summary>
@@ -431,10 +434,27 @@ public bool ResultsDirectorySet
431434
/// </summary>
432435
public string? TestCaseFilter { get; private set; }
433436

437+
/// <summary>
434438
/// Path to dotnet executable to be used to invoke testhost.dll. Specifying this will skip looking up testhost.exe and will force usage of the testhost.dll.
435439
/// </summary>
436440
public string? DotnetHostPath { get; private set; }
437441

442+
/// <summary>
443+
/// When true, we capture standard output of child processes. When false the standard output is not captured and it will end up in command line.
444+
/// This makes the output visible to the user when running in vstest.console in-process. Such setup makes the behavior the same as in 17.6.3 and earlier.
445+
///
446+
/// The recommended way is to use this with ForwardStandardOutput=true to forward output as informational messages so the output is always visible in console and VS,
447+
/// unless the logging level is set to Warning or higher.
448+
///
449+
/// Lastly this can be used with ForwardStandardOutput=false, to suppress the output in console, which is behavior of 17.7.0 till now.
450+
/// </summary>
451+
public bool CaptureStandardOutput { get; private set; }
452+
453+
/// <summary>
454+
/// Forward captured standard output of testhost as Informational test messages. Default is true. Needs CaptureStandardOutput to be true.
455+
/// </summary>
456+
public bool ForwardStandardOutput { get; private set; }
457+
438458
/// <inheritdoc/>
439459
public override XmlElement ToXml()
440460
{
@@ -550,6 +570,14 @@ public override XmlElement ToXml()
550570
root.AppendChild(treatAsError);
551571
}
552572

573+
XmlElement captureStandardOutput = doc.CreateElement(nameof(CaptureStandardOutput));
574+
captureStandardOutput.InnerXml = CaptureStandardOutput.ToString();
575+
root.AppendChild(captureStandardOutput);
576+
577+
XmlElement forwardStandardOutput = doc.CreateElement(nameof(ForwardStandardOutput));
578+
forwardStandardOutput.InnerXml = ForwardStandardOutput.ToString();
579+
root.AppendChild(forwardStandardOutput);
580+
553581
return root;
554582
}
555583

@@ -907,6 +935,35 @@ public static RunConfiguration FromXml(XmlReader reader)
907935
case "TargetFrameworkTestHostDemultiplexer":
908936
reader.Skip();
909937
break;
938+
939+
case "CaptureStandardOutput":
940+
XmlRunSettingsUtilities.ThrowOnHasAttributes(reader);
941+
string captureStandardOutputStr = reader.ReadElementContentAsString();
942+
943+
bool bCaptureStandardOutput;
944+
if (!bool.TryParse(captureStandardOutputStr, out bCaptureStandardOutput))
945+
{
946+
throw new SettingsException(string.Format(CultureInfo.CurrentCulture,
947+
Resources.Resources.InvalidSettingsIncorrectValue, Constants.RunConfigurationSettingsName, bCaptureStandardOutput, elementName));
948+
}
949+
950+
runConfiguration.CaptureStandardOutput = bCaptureStandardOutput;
951+
break;
952+
953+
case "ForwardStandardOutput":
954+
XmlRunSettingsUtilities.ThrowOnHasAttributes(reader);
955+
string forwardStandardOutputStr = reader.ReadElementContentAsString();
956+
957+
bool bForwardStandardOutput;
958+
if (!bool.TryParse(forwardStandardOutputStr, out bForwardStandardOutput))
959+
{
960+
throw new SettingsException(string.Format(CultureInfo.CurrentCulture,
961+
Resources.Resources.InvalidSettingsIncorrectValue, Constants.RunConfigurationSettingsName, bForwardStandardOutput, elementName));
962+
}
963+
964+
runConfiguration.ForwardStandardOutput = bForwardStandardOutput;
965+
break;
966+
910967
default:
911968
// Ignore a runsettings element that we don't understand. It could occur in the case
912969
// the test runner is of a newer version, but the test host is of an earlier version.

‎src/Microsoft.TestPlatform.ObjectModel/TestProperty/TestProperty.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ public class TestProperty : IEquatable<TestProperty>
1919
{
2020
private static readonly ConcurrentDictionary<string, Type> TypeCache = new();
2121

22-
private static bool DisableFastJson { get; set; } = FeatureFlag.Instance.IsSet(FeatureFlag.DISABLE_FASTER_JSON_SERIALIZATION);
22+
private static bool DisableFastJson { get; set; } = FeatureFlag.Instance.IsSet(FeatureFlag.VSTEST_DISABLE_FASTER_JSON_SERIALIZATION);
2323

2424
private Type _valueType;
2525

‎src/Microsoft.TestPlatform.TestHostProvider/Hosting/DefaultTestHostManager.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ public class DefaultTestHostManager : ITestRuntimeProvider2
6666
private StringBuilder? _testHostProcessStdError;
6767
private StringBuilder? _testHostProcessStdOut;
6868
private IMessageLogger? _messageLogger;
69+
private bool _captureOutput;
6970
private bool _hostExitedEventRaised;
7071
private TestHostManagerCallbacks? _testHostManagerCallbacks;
7172

@@ -371,7 +372,9 @@ public void Initialize(IMessageLogger? logger, string runsettingsXml)
371372
var runConfiguration = XmlRunSettingsUtilities.GetRunConfigurationNode(runsettingsXml);
372373

373374
_messageLogger = logger;
374-
_testHostManagerCallbacks = new TestHostManagerCallbacks(_environmentVariableHelper.GetEnvironmentVariable("VSTEST_EXPERIMENTAL_FORWARD_OUTPUT_FEATURE") == "1", logger);
375+
_captureOutput = runConfiguration.CaptureStandardOutput;
376+
var forwardOutput = runConfiguration.ForwardStandardOutput;
377+
_testHostManagerCallbacks = new TestHostManagerCallbacks(forwardOutput, logger);
375378
_architecture = runConfiguration.TargetPlatform;
376379
_targetFramework = runConfiguration.TargetFramework;
377380
_testHostProcess = null;
@@ -528,14 +531,15 @@ private bool LaunchHost(TestProcessStartInfo testHostStartInfo, CancellationToke
528531
{
529532
EqtTrace.Verbose("DefaultTestHostManager: Starting process '{0}' with command line '{1}'", testHostStartInfo.FileName, testHostStartInfo.Arguments);
530533
cancellationToken.ThrowIfCancellationRequested();
534+
var outputCallback = _captureOutput ? OutputReceivedCallback : null;
531535
_testHostProcess = _processHelper.LaunchProcess(
532536
testHostStartInfo.FileName!,
533537
testHostStartInfo.Arguments,
534538
testHostStartInfo.WorkingDirectory,
535539
testHostStartInfo.EnvironmentVariables,
536540
ErrorReceivedCallback,
537541
ExitCallBack,
538-
OutputReceivedCallback) as Process;
542+
outputCallback) as Process;
539543
}
540544
else
541545
{

0 commit comments

Comments
 (0)