Fixing nunit inconclusive tests reported as failure (#533)

* Fixing up the mappings of NotRun tests in trx logger and printing it out on console. This is a fix ported from TPV1. * Unrelated issue to this change but this is failing build.
microsoft · Feb 22, 2017 · 05027bd · 05027bd
1 parent 035d1bf
commit 05027bd
Show file tree

Hide file tree

Showing 21 changed files with 184 additions and 20 deletions.
diff --git a/src/Microsoft.TestPlatform.Extensions.TrxLogger/Utility/Converter.cs b/src/Microsoft.TestPlatform.Extensions.TrxLogger/Utility/Converter.cs
@@ -141,13 +141,9 @@ internal static TrxObjectModel.TestOutcome ToOutcome(ObjectModel.TestOutcome roc
                     outcome = TrxObjectModel.TestOutcome.Passed;
                     break;
                 case ObjectModel.TestOutcome.Skipped:
-                    outcome = TrxObjectModel.TestOutcome.NotExecuted;
-                    break;
                 case ObjectModel.TestOutcome.None:
-                    outcome = TrxObjectModel.TestOutcome.Failed;
-                    break;
                 case ObjectModel.TestOutcome.NotFound:
-                    outcome = TrxObjectModel.TestOutcome.Failed;
+                    outcome = TrxObjectModel.TestOutcome.NotExecuted;
                     break;
                 default:
                     Debug.Fail("Unexpected Outcome.");

diff --git a/src/vstest.console/Internal/ConsoleLogger.cs b/src/vstest.console/Internal/ConsoleLogger.cs
@@ -331,6 +331,12 @@ private void TestResultHandler(object sender, TestResultEventArgs e)
                 }
                 this.testsPassed++;
             }
+            else
+            {
+                string output = string.Format(CultureInfo.CurrentCulture, CommandLineResources.NotRunTestIndicator, name);
+                Output.WriteLine(output, OutputLevel.Information);
+                DisplayFullInformation(e.Result);
+            }
         }
 
         /// <summary>

diff --git a/src/vstest.console/Resources/Resources.Designer.cs b/src/vstest.console/Resources/Resources.Designer.cs
diff --git a/src/vstest.console/Resources/Resources.resx b/src/vstest.console/Resources/Resources.resx
@@ -637,4 +637,7 @@
   <data name="InIsolationDeprecated" xml:space="preserve">
     <value>The /InIsolation flag is deprecated. The tests are always run in a separate process</value>
   </data>
+  <data name="NotRunTestIndicator" xml:space="preserve">
+    <value>Not Run  {0}</value>
+  </data>
 </root>
diff --git a/src/vstest.console/Resources/xlf/Resources.cs.xlf b/src/vstest.console/Resources/xlf/Resources.cs.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.de.xlf b/src/vstest.console/Resources/xlf/Resources.de.xlf
@@ -1519,6 +1519,11 @@ Standardeinstellungen werden verwendet, wenn die Verwendung der Einstellungsdate
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.es.xlf b/src/vstest.console/Resources/xlf/Resources.es.xlf
@@ -1531,6 +1531,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.fr.xlf b/src/vstest.console/Resources/xlf/Resources.fr.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.it.xlf b/src/vstest.console/Resources/xlf/Resources.it.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.ja.xlf b/src/vstest.console/Resources/xlf/Resources.ja.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.ko.xlf b/src/vstest.console/Resources/xlf/Resources.ko.xlf
@@ -1520,6 +1520,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.pl.xlf b/src/vstest.console/Resources/xlf/Resources.pl.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf b/src/vstest.console/Resources/xlf/Resources.pt-BR.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.ru.xlf b/src/vstest.console/Resources/xlf/Resources.ru.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.tr.xlf b/src/vstest.console/Resources/xlf/Resources.tr.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.xlf b/src/vstest.console/Resources/xlf/Resources.xlf
@@ -89,13 +89,13 @@
       </trans-unit>
       <trans-unit id="EnableLoggersArgumentHelp">
         <source>--logger|/logger:&lt;Logger Uri/FriendlyName&gt;
-          Specify a logger for test results.  For example, to log results into a
-          Visual Studio Test Results File (TRX) use  /logger:trx [;LogFileName=&lt;Defaults to unique file name&gt;]
-          Creates file in TestResults directory with given LogFileName.
-          
-          To publish test results to Team Foundation Server, use TfsPublisher as shown below
-          Example: /logger:TfsPublisher;
-          Collection=&lt;team project collection url&gt;;
+      Specify a logger for test results.  For example, to log results into a 
+      Visual Studio Test Results File (TRX) use  /logger:trx [;LogFileName=&lt;Defaults to unique file name&gt;]
+      Creates file in TestResults directory with given LogFileName.
+
+      To publish test results to Team Foundation Server, use TfsPublisher as shown below
+      Example: /logger:TfsPublisher;
+                Collection=&lt;team project collection url&gt;;
                 BuildName=&lt;build name&gt;;
                 TeamProject=&lt;team project name&gt;
                 [;Platform=&lt;Defaults to "Any CPU"&gt;]
@@ -636,6 +636,47 @@
         <target state="new">Timeout to connect or process request for DesignModeClient on port: {0}</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
+      <trans-unit id="CLIRunSettingsArgumentHelp">
+        <source>Args:
+      Any extra arguments that should be passed to adapter. Arguments may be specified as name-value pair of the form &lt;n&gt;=&lt;v&gt;, where &lt;n&gt; is the argument name, and &lt;v&gt; is the argument value. Use a space to separate multiple arguments.</source>
+        <target state="new">Args:
+      Any extra arguments that should be passed to adapter. Arguments may be specified as name-value pair of the form &lt;n&gt;=&lt;v&gt;, where &lt;n&gt; is the argument name, and &lt;v&gt; is the argument value. Use a space to separate multiple arguments.</target>
+        <note></note>
+      </trans-unit>
+      <trans-unit id="MalformedRunSettingsKey">
+        <source>One or more runsettings provided contain invalid token</source>
+        <target state="new">One or more runsettings provided contain invalid token</target>
+        <note></note>
+      </trans-unit>
+      <trans-unit id="ResultsDirectoryArgumentHelp">
+        <source>--ResultsDirectory|/ResultsDirectory
+      Test results directory will be created in specified path if not exists.
+      Example  /ResultsDirectory:&lt;pathToResultsDirectory&gt;</source>
+        <target state="new">--ResultsDirectory|/ResultsDirectory
+      Test results directory will be created in specified path if not exists.
+      Example  /ResultsDirectory:&lt;pathToResultsDirectory&gt;</target>
+        <note></note>
+      </trans-unit>
+      <trans-unit id="ResultsDirectoryValueRequired">
+        <source>The /ResultsDirectory parameter requires a value, where the test results should be saved. Example:  /ResultsDirectory:c:\MyTestResultsDirectory</source>
+        <target state="new">The /ResultsDirectory parameter requires a value, where the test results should be saved. Example:  /ResultsDirectory:c:\MyTestResultsDirectory</target>
+        <note></note>
+      </trans-unit>
+      <trans-unit id="InvalidResultsDirectoryPathCommand">
+        <source>The path '{0}' specified in the 'ResultsDirectory' is invalid. Error: {1}</source>
+        <target state="new">The path '{0}' specified in the 'ResultsDirectory' is invalid. Error: {1}</target>
+        <note></note>
+      </trans-unit>
+      <trans-unit id="InIsolationDeprecated">
+        <source>The /InIsolation flag is deprecated. The tests are always run in a separate process</source>
+        <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf b/src/vstest.console/Resources/xlf/Resources.zh-Hans.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf b/src/vstest.console/Resources/xlf/Resources.zh-Hant.xlf
@@ -1519,6 +1519,11 @@
         <target state="new">The /InIsolation flag is deprecated. The tests are always run in a separate process</target>
         <note></note>
       </trans-unit>
+      <trans-unit id="NotRunTestIndicator">
+        <source>Not Run  {0}</source>
+        <target state="new">Not Run  {0}</target>
+        <note></note>
+      </trans-unit>
     </body>
   </file>
 </xliff>
diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/TrxLoggerTests.cs
@@ -25,6 +25,7 @@ namespace Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests
     using ObjectModel = Microsoft.VisualStudio.TestPlatform.ObjectModel;
     using TrxLoggerObjectModel = Microsoft.TestPlatform.Extensions.TrxLogger.ObjectModel;
     using TrxLoggerResources = Microsoft.VisualStudio.TestPlatform.Extensions.TrxLogger.Resources.TrxResource;
+    using Microsoft.TestPlatform.Extensions.TrxLogger.Utility;
 
     [TestClass]
     public class TrxLoggerTests

diff --git a/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Utility/ConverterTests.cs b/test/Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests/Utility/ConverterTests.cs
@@ -0,0 +1,45 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace Microsoft.TestPlatform.Extensions.TrxLogger.UnitTests.Utility
+{
+    using Microsoft.TestPlatform.Extensions.TrxLogger.Utility;
+    using Microsoft.VisualStudio.TestPlatform.ObjectModel;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    using TrxLoggerOutcome = Microsoft.TestPlatform.Extensions.TrxLogger.ObjectModel.TestOutcome;
+
+    [TestClass]
+    public class ConverterTests
+    {
+        [TestMethod]
+        public void ToOutcomeShouldMapFailedToFailed()
+        {
+            Assert.AreEqual(TrxLoggerOutcome.Failed, Converter.ToOutcome(TestOutcome.Failed));
+        }
+
+        [TestMethod]
+        public void ToOutcomeShouldMapPassedToPassed()
+        {
+            Assert.AreEqual(TrxLoggerOutcome.Passed, Converter.ToOutcome(TestOutcome.Passed));
+        }
+
+        [TestMethod]
+        public void ToOutcomeShouldMapSkippedToNotExecuted()
+        {
+            Assert.AreEqual(TrxLoggerOutcome.NotExecuted, Converter.ToOutcome(TestOutcome.Skipped));
+        }
+
+        [TestMethod]
+        public void ToOutcomeShouldMapNoneToNotExecuted()
+        {
+            Assert.AreEqual(TrxLoggerOutcome.NotExecuted, Converter.ToOutcome(TestOutcome.None));
+        }
+
+        [TestMethod]
+        public void ToOutcomeShouldMapNotFoundToNotExecuted()
+        {
+            Assert.AreEqual(TrxLoggerOutcome.NotExecuted, Converter.ToOutcome(TestOutcome.NotFound));
+        }
+    }
+}
diff --git a/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs b/test/vstest.console.UnitTests/Internal/ConsoleLoggerTests.cs
@@ -157,10 +157,8 @@ public void TestResultHandlerShouldWriteToConsoleIfTestResultEventsAreRaised()
 
             this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.PassedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
             this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.FailedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
+            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.PassedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
+            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.NotRunTestIndicator, "TestName"), OutputLevel.Information), Times.Exactly(2));
         }
 
         [TestMethod]
@@ -176,11 +174,9 @@ public void TestResultHandlerShouldWriteToConsoleButSkipPassedTestsForMinimalVer
             this.testRunRequest.Raise(m => m.OnRunStatsChange += null, eventArgs);
             this.FlushLoggerMessages();
 
+            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.PassedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
             this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.FailedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
-            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.SkippedTestIndicator, "TestName"), OutputLevel.Information), Times.Once());
+            this.mockOutput.Verify(o => o.WriteLine(string.Format(CultureInfo.CurrentCulture, CommandLineResources.NotRunTestIndicator, "TestName"), OutputLevel.Information), Times.Exactly(2));
         }
 
         [TestMethod]