Add StandardOutputProperty and StandardErrorProperty (#3748) (#3749)

microsoft · Sep 3, 2024 · ccf7de7 · ccf7de7
1 parent 26bf645
commit ccf7de7
Show file tree

Hide file tree

Showing 10 changed files with 155 additions and 13 deletions.
diff --git a/...tingplatform/Source/TestingPlatformExplorer/TestingFramework/TestingFramework.Services.cs b/...tingplatform/Source/TestingPlatformExplorer/TestingFramework/TestingFramework.Services.cs
@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System.Text;
+
+namespace TestingPlatformExplorer.TestingFramework;
+
+public interface ITestOutputHelper
+{
+    void WriteLine(string message);
+    void WriteErrorLine(string message);
+}
+
+internal sealed class TestOutputHelper : ITestOutputHelper
+{
+    public StringBuilder Output { get; set; } = new StringBuilder();
+    public StringBuilder Error { get; set; } = new StringBuilder();
+
+    public void WriteErrorLine(string message) => Error.AppendLine(message);
+
+    public void WriteLine(string message) => Output.AppendLine(message);
+}
diff --git a/docs/testingplatform/Source/TestingPlatformExplorer/TestingFramework/TestingFramework.cs b/docs/testingplatform/Source/TestingPlatformExplorer/TestingFramework/TestingFramework.cs
@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
+
 using System.Globalization;
 using System.Reflection;
 using System.Text;
@@ -16,8 +18,6 @@
 using Microsoft.Testing.Platform.OutputDevice;
 using Microsoft.Testing.Platform.Requests;
 
-using static System.Net.Mime.MediaTypeNames;
-
 namespace TestingPlatformExplorer.TestingFramework;
 
 internal sealed class TestingFramework : ITestFramework, IDataProducer, IDisposable, IOutputDeviceDataProducer
@@ -181,10 +181,19 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                                 await _dop.WaitAsync();
                                 try
                                 {
+                                    var testOutputHelper = new TestOutputHelper();
                                     object? instance = Activator.CreateInstance(test.DeclaringType!);
                                     try
                                     {
-                                        test.Invoke(instance, null);
+                                        if (test.GetParameters().Length == 1 && test.GetParameters()[0].ParameterType == typeof(ITestOutputHelper))
+                                        {
+                                            test.Invoke(instance, new object[] { testOutputHelper });
+
+                                        }
+                                        else
+                                        {
+                                            test.Invoke(instance, null);
+                                        }
 
                                         var successfulTestNode = new TestNode()
                                         {
@@ -198,6 +207,16 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                                             FillTrxProperties(successfulTestNode, test);
                                         }
 
+                                        if (testOutputHelper.Output.Length > 0)
+                                        {
+                                            successfulTestNode.Properties.Add(new StandardOutputProperty(testOutputHelper.Output.ToString()));
+                                        }
+
+                                        if (testOutputHelper.Error.Length > 0)
+                                        {
+                                            successfulTestNode.Properties.Add(new StandardErrorProperty(testOutputHelper.Error.ToString()));
+                                        }
+
                                         await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(runTestExecutionRequest.Session.SessionUid, successfulTestNode));
 
                                         lock (reportBody)
@@ -219,6 +238,16 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                                             FillTrxProperties(assertionFailedTestNode, test, assertionException);
                                         }
 
+                                        if (testOutputHelper.Output.Length > 0)
+                                        {
+                                            assertionFailedTestNode.Properties.Add(new StandardOutputProperty(testOutputHelper.Output.ToString()));
+                                        }
+
+                                        if (testOutputHelper.Error.Length > 0)
+                                        {
+                                            assertionFailedTestNode.Properties.Add(new StandardErrorProperty(testOutputHelper.Error.ToString()));
+                                        }
+
                                         await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(runTestExecutionRequest.Session.SessionUid, assertionFailedTestNode));
 
                                         reportBody.AppendLine(CultureInfo.InvariantCulture, $"Test {assertionFailedTestNode.Uid} failed");
@@ -237,6 +266,16 @@ public async Task ExecuteRequestAsync(ExecuteRequestContext context)
                                             FillTrxProperties(failedTestNode, test, ex);
                                         }
 
+                                        if (testOutputHelper.Output.Length > 0)
+                                        {
+                                            failedTestNode.Properties.Add(new StandardOutputProperty(testOutputHelper.Output.ToString()));
+                                        }
+
+                                        if (testOutputHelper.Error.Length > 0)
+                                        {
+                                            failedTestNode.Properties.Add(new StandardErrorProperty(testOutputHelper.Error.ToString()));
+                                        }
+
                                         await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(runTestExecutionRequest.Session.SessionUid, failedTestNode));
 
                                         lock (reportBody)

diff --git a/docs/testingplatform/Source/TestingPlatformExplorer/UnitTests.cs b/docs/testingplatform/Source/TestingPlatformExplorer/UnitTests.cs
@@ -14,13 +14,23 @@ public class SomeTests
     public static void TestMethod2() => Assert.AreEqual(1, 2);
 
     [TestMethod]
-    public static void TestMethod3()
+    public static void TestMethod3(ITestOutputHelper testOutputHelper)
     {
-        int a = 1;
-        int b = 0;
-        int c = a / b;
+        testOutputHelper.WriteLine("I'm running TestMethod3");
 
-        Assert.AreEqual(c, 2);
+        try
+        {
+            int a = 1;
+            int b = 0;
+            int c = a / b;
+
+            Assert.AreEqual(c, 2);
+        }
+        catch (Exception ex)
+        {
+            testOutputHelper.WriteErrorLine(ex.ToString());
+            throw;
+        }
     }
 
     [Skip]

diff --git a/src/Platform/Microsoft.Testing.Platform/Messages/TestNodeProperties.cs b/src/Platform/Microsoft.Testing.Platform/Messages/TestNodeProperties.cs
@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
-namespace Microsoft.Testing.Platform.Extensions.Messages;
+using System.Diagnostics.CodeAnalysis;
 
-#pragma warning disable SA1201 // Elements should appear in the correct order
+namespace Microsoft.Testing.Platform.Extensions.Messages;
 
 /// <summary>
 /// The interface that every test node property must implement.
@@ -312,6 +312,20 @@ public sealed record TestMethodIdentifierProperty(string AssemblyFullName, strin
 /// <param name="Value">Value name.</param>
 public sealed record TestMetadataProperty(string Key, string Value) : IProperty;
 
+/// <summary>
+/// Property that represents standard output to associate with a test node.
+/// </summary>
+/// <param name="StandardOutput">The standard output.</param>
+[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")]
+public record StandardOutputProperty(string StandardOutput) : IProperty;
+
+/// <summary>
+/// Property that represents standard error to associate with a test node.
+/// </summary>
+/// <param name="StandardError">The standard error.</param>
+[Experimental("TPEXP", UrlFormat = "https://aka.ms/testingplatform/diagnostics#{0}")]
+public record StandardErrorProperty(string StandardError) : IProperty;
+
 internal sealed record SerializableKeyValuePairStringProperty(string Key, string Value) : KeyValuePairStringProperty(Key, Value);
 
 internal sealed record SerializableNamedKeyValuePairsStringProperty(string Name, KeyValuePair<string, string>[] Pairs) : IProperty;

diff --git a/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/PublicAPI/PublicAPI.Unshipped.txt
@@ -1,4 +1,34 @@
 #nullable enable
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.get -> string!
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutput.init -> void
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.EqualityContract.get -> System.Type!
+[TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.ToString() -> string!
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.PrintMembers(System.Text.StringBuilder! builder) -> bool
+[TPEXP]static Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.operator !=(Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty? left, Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty? right) -> bool
+[TPEXP]static Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.operator ==(Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty? left, Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty? right) -> bool
+[TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.GetHashCode() -> int
+[TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.Equals(object? obj) -> bool
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.<Clone>$() -> Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty!
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.Equals(Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty? other) -> bool
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutputProperty(Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty! original) -> void
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.Deconstruct(out string! StandardOutput) -> void
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardOutputProperty.StandardOutputProperty(string! StandardOutput) -> void
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.StandardError.get -> string!
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.StandardError.init -> void
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.EqualityContract.get -> System.Type!
+[TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.ToString() -> string!
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.PrintMembers(System.Text.StringBuilder! builder) -> bool
+[TPEXP]static Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.operator !=(Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty? left, Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty? right) -> bool
+[TPEXP]static Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.operator ==(Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty? left, Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty? right) -> bool
+[TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.GetHashCode() -> int
+[TPEXP]override Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.Equals(object? obj) -> bool
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.<Clone>$() -> Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty!
+[TPEXP]virtual Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.Equals(Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty? other) -> bool
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.StandardErrorProperty(Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty! original) -> void
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.Deconstruct(out string! StandardError) -> void
+[TPEXP]Microsoft.Testing.Platform.Extensions.Messages.StandardErrorProperty.StandardErrorProperty(string! StandardError) -> void
 Microsoft.Testing.Platform.OutputDevice.FormattedTextOutputDeviceData.Padding.get -> int?
 Microsoft.Testing.Platform.OutputDevice.FormattedTextOutputDeviceData.Padding.init -> void
 [TPEXP]Microsoft.Testing.Platform.Capabilities.TestFramework.TestFrameworkCapabilitiesExtensions

diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/Json/Json.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/Json/Json.cs
@@ -176,6 +176,16 @@ public Json(Dictionary<Type, JsonSerializer>? serializers = null, Dictionary<Typ
                     continue;
                 }
 
+                if (property is StandardOutputProperty standardOutputProperty)
+                {
+                    properties.Add(("standardOutput", standardOutputProperty.StandardOutput));
+                }
+
+                if (property is StandardErrorProperty standardErrorProperty)
+                {
+                    properties.Add(("standardError", standardErrorProperty.StandardError));
+                }
+
                 if (property is TestNodeStateProperty testNodeStateProperty)
                 {
                     properties.Add(("node-type", "action"));

diff --git a/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/SerializerUtilities.cs b/src/Platform/Microsoft.Testing.Platform/ServerMode/JsonRpc/SerializerUtilities.cs
@@ -249,6 +249,18 @@ static SerializerUtilities()
                         continue;
                     }
 
+                    if (property is StandardOutputProperty consoleStandardOutputProperty)
+                    {
+                        properties["standardOutput"] = consoleStandardOutputProperty.StandardOutput;
+                        continue;
+                    }
+
+                    if (property is StandardErrorProperty standardErrorProperty)
+                    {
+                        properties["standardError"] = standardErrorProperty.StandardError;
+                        continue;
+                    }
+
                     if (property is TestNodeStateProperty testNodeStateProperty)
                     {
                         properties["node-type"] = "action";

diff --git a/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs b/test/IntegrationTests/MSTest.Acceptance.IntegrationTests/SdkTests.cs
@@ -219,6 +219,7 @@ public async Task RunTests_With_MSTestRunner_Standalone_Selectively_Enabled_Exte
         }
     }
 
+    [NonTest]
     [ArgumentsProvider(nameof(GetBuildMatrixMultiTfmFoldedBuildConfiguration))]
     public async Task RunTests_With_MSTestRunner_Standalone_EnableAll_Extensions(string multiTfm, BuildConfiguration buildConfiguration)
     {

diff --git a/...tionTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs b/...tionTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/MSBuildTests.Solution.cs
@@ -27,7 +27,7 @@ public MSBuildTests_Solution(ITestExecutionContext testExecutionContext, Accepta
     private void CheckPatch()
     {
         // https://github.com/dotnet/sdk/issues/37712
-        if (DateTime.UtcNow.Date > new DateTime(2024, 9, 1))
+        if (DateTime.UtcNow.Date > new DateTime(2024, 12, 1))
         {
             throw new InvalidOperationException("Check if we can remove the patch!");
         }

diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/ServerMode/FormatterUtilitiesTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/ServerMode/FormatterUtilitiesTests.cs
@@ -1,6 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
+#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed.
+
 using Microsoft.Testing.Platform.Extensions.Messages;
 using Microsoft.Testing.Platform.ServerMode;
 
@@ -168,7 +170,7 @@ private static void AssertSerialize(Type type, string instanceSerialized)
 
         if (type == typeof(TestNode))
         {
-            Assert.AreEqual("""{"uid":"uid","display-name":"DisplayName","tuples-key":[{"a":"1"},{"b":"2"}],"array-key":["1","2"],"time.start-utc":"2023-01-01T01:01:01.0000000+00:00","time.stop-utc":"2023-01-01T01:01:01.0000000+00:00","time.duration-ms":0,"location.namespace":"namespace","location.type":"typeName","location.method":"methodName(param1,param2)","location.file":"filePath","location.line-start":1,"location.line-end":2,"key":"value","node-type":"action","execution-state":"failed","error.message":"sample","error.stacktrace":"","assert.actual":"","assert.expected":""}""".Replace(" ", string.Empty), instanceSerialized, because);
+            Assert.AreEqual("""{"uid":"uid","display-name":"DisplayName","tuples-key":[{"a":"1"},{"b":"2"}],"array-key":["1","2"],"standardError":"textProperty2","standardOutput":"textProperty","time.start-utc":"2023-01-01T01:01:01.0000000+00:00","time.stop-utc":"2023-01-01T01:01:01.0000000+00:00","time.duration-ms":0,"location.namespace":"namespace","location.type":"typeName","location.method":"methodName(param1,param2)","location.file":"filePath","location.line-start":1,"location.line-end":2,"key":"value","node-type":"action","execution-state":"failed","error.message":"sample","error.stacktrace":"","assert.actual":"","assert.expected":""}""".Replace(" ", string.Empty), instanceSerialized, because);
             return;
         }
 
@@ -180,7 +182,7 @@ private static void AssertSerialize(Type type, string instanceSerialized)
 
         if (type == typeof(TestNodeUpdateMessage))
         {
-            Assert.AreEqual("""{"node":{"uid":"uid","display-name":"DisplayName","tuples-key":[{"a":"1"},{"b":"2"}],"array-key":["1","2"],"time.start-utc":"2023-01-01T01:01:01.0000000+00:00","time.stop-utc":"2023-01-01T01:01:01.0000000+00:00","time.duration-ms":0,"location.namespace":"namespace","location.type":"typeName","location.method":"methodName(param1,param2)","location.file":"filePath","location.line-start":1,"location.line-end":2,"key":"value","node-type":"action","execution-state":"failed","error.message":"sample","error.stacktrace":"","assert.actual":"","assert.expected":""},"parent":"parent-uid"}""".Replace(" ", string.Empty), instanceSerialized, because);
+            Assert.AreEqual("""{"node":{"uid":"uid","display-name":"DisplayName","tuples-key":[{"a":"1"},{"b":"2"}],"array-key":["1","2"],"standardError":"textProperty2","standardOutput":"textProperty","time.start-utc":"2023-01-01T01:01:01.0000000+00:00","time.stop-utc":"2023-01-01T01:01:01.0000000+00:00","time.duration-ms":0,"location.namespace":"namespace","location.type":"typeName","location.method":"methodName(param1,param2)","location.file":"filePath","location.line-start":1,"location.line-end":2,"key":"value","node-type":"action","execution-state":"failed","error.message":"sample","error.stacktrace":"","assert.actual":"","assert.expected":""},"parent":"parent-uid"}""".Replace(" ", string.Empty), instanceSerialized, because);
             return;
         }
 
@@ -430,6 +432,8 @@ static TestNode GetSampleTestNode()
             testNode.Properties.Add(new TestMethodIdentifierProperty("assemblyFullName", "namespace", "typeName", "methodName", ["param1", "param2"], "returnTypeFullName"));
             testNode.Properties.Add(new TimingProperty(new TimingInfo(new DateTimeOffset(2023, 01, 01, 01, 01, 01, TimeSpan.Zero), new DateTimeOffset(2023, 01, 01, 01, 01, 01, TimeSpan.Zero), TimeSpan.Zero)));
             testNode.Properties.Add(new FailedTestNodeStateProperty(new InvalidOperationException("sample")));
+            testNode.Properties.Add(new StandardOutputProperty("textProperty"));
+            testNode.Properties.Add(new StandardErrorProperty("textProperty2"));
             testNode.Properties.Add(new SerializableNamedArrayStringProperty("array-key", ["1", "2"]));
             testNode.Properties.Add(new SerializableNamedKeyValuePairsStringProperty("tuples-key", [new KeyValuePair<string, string>("a", "1"), new KeyValuePair<string, string>("b", "2"),]));