Add FSharp E2E test (#683)

* Add FSharp E2E test * More representative method name * Improve test * Fix string expand * Fix passed tests detection * Fix typo in tests * Comment grammar * Review feedback Co-authored-by: nohwnd <me@jakubjares.com>
microsoft · Mar 20, 2020 · 3c76ad3 · 3c76ad3
1 parent efa1a6d
commit 3c76ad3
Show file tree

Hide file tree

Showing 9 changed files with 113 additions and 11 deletions.
diff --git a/TestFx.sln b/TestFx.sln
@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio 15
-VisualStudioVersion = 15.0.26228.9
+# Visual Studio Version 16
+VisualStudioVersion = 16.0.29728.190
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{FF8B1B72-55A1-4FFE-809E-7B79323ED8D0}"
 EndProject
@@ -185,6 +185,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeploymentTestProjectNetCor
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProjectNetCore", "test\E2ETests\TestAssets\TimeoutTestProjectNetCore\TimeoutTestProjectNetCore.csproj", "{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}"
 EndProject
+Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharpTestProject", "test\E2ETests\TestAssets\FSharpTestProject\FSharpTestProject.fsproj", "{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}"
+EndProject
 Global
 	GlobalSection(SharedMSBuildProjectFiles) = preSolution
 		src\TestFramework\Extension.Shared\Extension.Shared.projitems*{272ca5e1-8e81-4825-9e47-86cce02f700d}*SharedItemsImports = 13
@@ -1141,6 +1143,30 @@ Global
 		{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x64.Build.0 = Release|Any CPU
 		{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x86.ActiveCfg = Release|Any CPU
 		{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x86.Build.0 = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|ARM.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|ARM.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x64.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x86.Build.0 = Debug|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|ARM.ActiveCfg = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|ARM.Build.0 = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.ActiveCfg = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.Build.0 = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.ActiveCfg = Release|Any CPU
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1204,6 +1230,7 @@ Global
 		{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
 		{26F0B8EF-92D4-4A23-ACB4-D1B662F0EEBE} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
 		{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
+		{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}

diff --git a/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs b/src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs
@@ -24,8 +24,13 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string
             var testClassName = testCase.GetPropertyValue(Constants.TestClassNameProperty) as string;
             var declaringClassName = testCase.GetPropertyValue(Constants.DeclaringClassNameProperty) as string;
 
-            var parts = testCase.FullyQualifiedName.Split('.');
-            var name = parts[parts.Length - 1];
+            var fullyQualifiedName = testCase.FullyQualifiedName;
+
+            // Not using Replace because there can be multiple instances of that string.
+            var name = fullyQualifiedName.StartsWith($"{testClassName}.")
+                ? fullyQualifiedName.Remove(0, $"{testClassName}.".Length)
+                : fullyQualifiedName;
+
             TestMethod testMethod = new TestMethod(name, testClassName, source, isAsync);
 
             if (declaringClassName != null && declaringClassName != testClassName)

diff --git a/test/E2ETests/Automation.CLI/CLITestBase.cs b/test/E2ETests/Automation.CLI/CLITestBase.cs
@@ -75,6 +75,10 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "",
             // this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed
             vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll"));
             vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler);
+            if (this.runEventsHandler.Errors.Any())
+            {
+                throw new Exception($"Run failed with {this.runEventsHandler.Errors.Count} errors:{Environment.NewLine}{string.Join(Environment.NewLine, this.runEventsHandler.Errors)}");
+            }
         }
 
         /// <summary>
@@ -101,7 +105,7 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList)
             {
                 var flag = this.discoveryEventsHandler.Tests.Contains(test)
                            || this.discoveryEventsHandler.Tests.Contains(GetTestMethodName(test));
-                Assert.IsTrue(flag, "Test {0} does not appear in discovered tests list.", test);
+                Assert.IsTrue(flag, "Test '{0}' does not appear in discovered tests list.", test);
             }
 
             // Make sure only expected number of tests are discovered and not more.
@@ -170,12 +174,15 @@ public void ValidateSkippedTests(params string[] skippedTests)
         /// <remarks>Provide the full test name similar to this format SampleTest.TestCode.TestMethodPass.</remarks>
         public void ValidatePassedTestsContain(params string[] passedTests)
         {
+            var passedTestResults = this.runEventsHandler.PassedTests.ToList();
             foreach (var test in passedTests)
             {
-                var testFound = this.runEventsHandler.PassedTests.Any(
+                var testFound = passedTestResults.Any(
                     p => test.Equals(p.TestCase?.FullyQualifiedName)
-                         || test.Equals(p.DisplayName));
-                Assert.IsTrue(testFound, "Test {0} does not appear in passed tests list.", test);
+                         || test.Equals(p.DisplayName)
+                         || test.Equals(p.TestCase.DisplayName));
+
+                Assert.IsTrue(testFound, "Test '{0}' does not appear in passed tests list.", test);
             }
         }
 
@@ -195,7 +202,7 @@ public void ValidateFailedTestsContain(string source, bool validateStackTraceInf
             {
                 var testFound = this.runEventsHandler.FailedTests.FirstOrDefault(f => test.Equals(f.TestCase?.FullyQualifiedName) ||
                            test.Equals(f.DisplayName));
-                Assert.IsNotNull(testFound, "Test {0} does not appear in failed tests list.", test);
+                Assert.IsNotNull(testFound, "Test '{0}' does not appear in failed tests list.", test);
 
                 // Skipping this check for x64 as of now. https://github.com/Microsoft/testfx/issues/60 should fix this.
                 if (source.IndexOf("x64") == -1 && validateStackTraceInfo)
@@ -222,7 +229,7 @@ public void ValidateSkippedTestsContain(params string[] skippedTests)
             {
                 var testFound = this.runEventsHandler.SkippedTests.Any(s => test.Equals(s.TestCase.FullyQualifiedName) ||
                            test.Equals(s.DisplayName));
-                Assert.IsTrue(testFound, "Test {0} does not appear in skipped tests list.", test);
+                Assert.IsTrue(testFound, "Test '{0}' does not appear in skipped tests list.", test);
             }
         }
 

diff --git a/test/E2ETests/Automation.CLI/RunEventsHandler.cs b/test/E2ETests/Automation.CLI/RunEventsHandler.cs
@@ -25,13 +25,16 @@ public class RunEventsHandler : ITestRunEventsHandler
         /// </summary>
         public IList<TestResult> SkippedTests { get; private set; }
 
+        public IList<string> Errors { get; private set; }
+
         public double ElapsedTimeInRunningTests { get; private set; }
 
         public RunEventsHandler()
         {
             this.PassedTests = new List<TestResult>();
             this.FailedTests = new List<TestResult>();
             this.SkippedTests = new List<TestResult>();
+            this.Errors = new List<string>();
         }
 
         public void HandleLogMessage(TestMessageLevel level, string message)
@@ -45,6 +48,7 @@ public void HandleLogMessage(TestMessageLevel level, string message)
                     EqtTrace.Warning(message);
                     break;
                 case TestMessageLevel.Error:
+                    this.Errors.Add(message);
                     EqtTrace.Error(message);
                     break;
                 default:

diff --git a/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj b/test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
@@ -41,6 +41,7 @@
     <Compile Include="AssertExtensibilityTests.cs" />
     <Compile Include="TimeoutTests.cs" />
     <Compile Include="CompatTests.cs" />
+    <Compile Include="TestProjectFSharpTests.cs" />
     <Compile Include="CustomTestExecutionExtensibilityTests.cs" />
     <Compile Include="DataSourceTests.cs" />
     <Compile Include="DeploymentTests.cs" />

diff --git a/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs b/test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs
@@ -0,0 +1,23 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+namespace MSTestAdapter.Smoke.E2ETests
+{
+    using Microsoft.MSTestV2.CLIAutomation;
+    using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+    [TestClass]
+    public class TestProjectFSharpTests : CLITestBase
+    {
+        private const string TestAssembly = "FSharpTestProject.dll";
+
+        [TestMethod]
+        public void ExecuteCustomTestExtensibilityTests()
+        {
+            this.InvokeVsTestForExecution(new string[] { TestAssembly });
+            this.ValidateFailedTestsCount(0);
+            this.ValidatePassedTestsCount(1);
+            this.ValidatePassedTestsContain("Test method passing with a . in it");
+        }
+    }
+}
diff --git a/test/E2ETests/TestAssets/FSharpTestProject/FSharpTestProject.fsproj b/test/E2ETests/TestAssets/FSharpTestProject/FSharpTestProject.fsproj
@@ -0,0 +1,24 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TestFxRoot Condition="$(TestFxRoot) == ''">..\..\..\..\</TestFxRoot>
+  </PropertyGroup>
+
+  <PropertyGroup>
+    <TargetFramework>net452</TargetFramework>
+    <IsPackable>false</IsPackable>
+    <GenerateProgramFile>false</GenerateProgramFile>
+    <AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
+    <OutputPath>$(TestFxRoot)artifacts\TestAssets\</OutputPath>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\..\src\TestFramework\Extension.Desktop\Extension.Desktop.csproj" />
+    <ProjectReference Include="..\..\..\..\src\TestFramework\MSTest.Core\MSTest.Core.csproj" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <Compile Include="Tests.fs" />
+  </ItemGroup>
+
+</Project>
diff --git a/test/E2ETests/TestAssets/FSharpTestProject/Tests.fs b/test/E2ETests/TestAssets/FSharpTestProject/Tests.fs
@@ -0,0 +1,11 @@
+namespace FSharpTestProject
+
+open System
+open Microsoft.VisualStudio.TestTools.UnitTesting
+
+[<TestClass>]
+type UnitTest1 () =
+
+    [<TestMethod>]
+    member this.``Test method passing with a . in it`` () =
+        Assert.IsTrue(true);
diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestCaseExtensionsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Extensions/TestCaseExtensionsTests.cs
@@ -20,7 +20,7 @@ public class TestCaseExtensionsTests
         [TestMethod]
         public void ToUnitTestElementShouldReturnUnitTestElementWithFieldsSet()
         {
-            TestCase testCase = new TestCase("DummyClass.DummyMethod", new Uri("DummyUri", UriKind.Relative), Assembly.GetCallingAssembly().FullName);
+            TestCase testCase = new TestCase("DummyClassName.DummyMethod", new Uri("DummyUri", UriKind.Relative), Assembly.GetCallingAssembly().FullName);
             testCase.DisplayName = "DummyDisplayName";
             var testCategories = new[] { "DummyCategory" };