Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add FSharp E2E test #683

Merged
merged 9 commits into from
Mar 20, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 29 additions & 2 deletions TestFx.sln
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio 15
VisualStudioVersion = 15.0.26228.9
# Visual Studio Version 16
VisualStudioVersion = 16.0.29728.190
MinimumVisualStudioVersion = 10.0.40219.1
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{FF8B1B72-55A1-4FFE-809E-7B79323ED8D0}"
EndProject
Expand Down Expand Up @@ -185,6 +185,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeploymentTestProjectNetCor
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TimeoutTestProjectNetCore", "test\E2ETests\TestAssets\TimeoutTestProjectNetCore\TimeoutTestProjectNetCore.csproj", "{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}"
EndProject
Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "FSharpTestProject", "test\E2ETests\TestAssets\FSharpTestProject\FSharpTestProject.fsproj", "{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}"
EndProject
Global
GlobalSection(SharedMSBuildProjectFiles) = preSolution
src\TestFramework\Extension.Shared\Extension.Shared.projitems*{272ca5e1-8e81-4825-9e47-86cce02f700d}*SharedItemsImports = 13
Expand Down Expand Up @@ -1141,6 +1143,30 @@ Global
{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x64.Build.0 = Release|Any CPU
{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x86.ActiveCfg = Release|Any CPU
{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564}.Release|x86.Build.0 = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|Any CPU.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|ARM.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|ARM.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x64.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x64.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x86.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Code Analysis Debug|x86.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|ARM.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|ARM.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x64.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x64.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x86.ActiveCfg = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Debug|x86.Build.0 = Debug|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|Any CPU.Build.0 = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|ARM.ActiveCfg = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|ARM.Build.0 = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.ActiveCfg = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x64.Build.0 = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.ActiveCfg = Release|Any CPU
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D}.Release|x86.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -1204,6 +1230,7 @@ Global
{4F0B2ACF-1341-42AF-918C-669A6D5CEA2B} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
{26F0B8EF-92D4-4A23-ACB4-D1B662F0EEBE} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
{ED27A844-6870-4FE6-8FEF-3ABDD1ED6564} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
{E5E58613-82FC-44CD-B75F-4F1C7ED52D0D} = {D53BD452-F69F-4FB3-8B98-386EDA28A4C8}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {31E0F4D5-975A-41CC-933E-545B2201FAF9}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,13 @@ internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string
var testClassName = testCase.GetPropertyValue(Constants.TestClassNameProperty) as string;
var declaringClassName = testCase.GetPropertyValue(Constants.DeclaringClassNameProperty) as string;

var parts = testCase.FullyQualifiedName.Split('.');
var name = parts[parts.Length - 1];
var fullyQualifiedName = testCase.FullyQualifiedName;

// Not using Replace because there can be multiple instances of that string.
var name = fullyQualifiedName.StartsWith($"{testClassName}.")
? fullyQualifiedName.Remove(0, $"{testClassName}.".Length)
: fullyQualifiedName;

TestMethod testMethod = new TestMethod(name, testClassName, source, isAsync);

if (declaringClassName != null && declaringClassName != testClassName)
Expand Down
19 changes: 13 additions & 6 deletions test/E2ETests/Automation.CLI/CLITestBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,10 @@ public void InvokeVsTestForExecution(string[] sources, string runSettings = "",
// this step of Initializing extensions should not be required after this issue: https://github.com/Microsoft/vstest/issues/236 is fixed
vsTestConsoleWrapper.InitializeExtensions(Directory.GetFiles(this.GetTestAdapterPath(), "*TestAdapter.dll"));
vsTestConsoleWrapper.RunTests(sources, runSettingXml, new TestPlatformOptions { TestCaseFilter = testCaseFilter }, this.runEventsHandler);
if (this.runEventsHandler.Errors.Any())
{
throw new Exception($"Run failed with {this.runEventsHandler.Errors.Count} errors:{Environment.NewLine}{string.Join(Environment.NewLine, this.runEventsHandler.Errors)}");
nohwnd marked this conversation as resolved.
Show resolved Hide resolved
}
}

/// <summary>
Expand All @@ -101,7 +105,7 @@ public void ValidateDiscoveredTests(params string[] discoveredTestsList)
{
var flag = this.discoveryEventsHandler.Tests.Contains(test)
|| this.discoveryEventsHandler.Tests.Contains(GetTestMethodName(test));
Assert.IsTrue(flag, "Test {0} does not appear in discovered tests list.", test);
Assert.IsTrue(flag, "Test '{0}' does not appear in discovered tests list.", test);
}

// Make sure only expected number of tests are discovered and not more.
Expand Down Expand Up @@ -170,12 +174,15 @@ public void ValidateSkippedTests(params string[] skippedTests)
/// <remarks>Provide the full test name similar to this format SampleTest.TestCode.TestMethodPass.</remarks>
public void ValidatePassedTestsContain(params string[] passedTests)
{
var passedTestResults = this.runEventsHandler.PassedTests.ToList();
foreach (var test in passedTests)
{
var testFound = this.runEventsHandler.PassedTests.Any(
var testFound = passedTestResults.Any(
p => test.Equals(p.TestCase?.FullyQualifiedName)
|| test.Equals(p.DisplayName));
Assert.IsTrue(testFound, "Test {0} does not appear in passed tests list.", test);
|| test.Equals(p.DisplayName)
|| test.Equals(p.TestCase.DisplayName));

Assert.IsTrue(testFound, "Test '{0}' does not appear in passed tests list.", test);
}
}

Expand All @@ -195,7 +202,7 @@ public void ValidateFailedTestsContain(string source, bool validateStackTraceInf
{
var testFound = this.runEventsHandler.FailedTests.FirstOrDefault(f => test.Equals(f.TestCase?.FullyQualifiedName) ||
test.Equals(f.DisplayName));
Assert.IsNotNull(testFound, "Test {0} does not appear in failed tests list.", test);
Assert.IsNotNull(testFound, "Test '{0}' does not appear in failed tests list.", test);

// Skipping this check for x64 as of now. https://github.com/Microsoft/testfx/issues/60 should fix this.
if (source.IndexOf("x64") == -1 && validateStackTraceInfo)
Expand All @@ -222,7 +229,7 @@ public void ValidateSkippedTestsContain(params string[] skippedTests)
{
var testFound = this.runEventsHandler.SkippedTests.Any(s => test.Equals(s.TestCase.FullyQualifiedName) ||
test.Equals(s.DisplayName));
Assert.IsTrue(testFound, "Test {0} does not appear in skipped tests list.", test);
Assert.IsTrue(testFound, "Test '{0}' does not appear in skipped tests list.", test);
}
}

Expand Down
4 changes: 4 additions & 0 deletions test/E2ETests/Automation.CLI/RunEventsHandler.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,16 @@ public class RunEventsHandler : ITestRunEventsHandler
/// </summary>
public IList<TestResult> SkippedTests { get; private set; }

public IList<string> Errors { get; private set; }

public double ElapsedTimeInRunningTests { get; private set; }

public RunEventsHandler()
{
this.PassedTests = new List<TestResult>();
this.FailedTests = new List<TestResult>();
this.SkippedTests = new List<TestResult>();
this.Errors = new List<string>();
}

public void HandleLogMessage(TestMessageLevel level, string message)
Expand All @@ -45,6 +48,7 @@ public void HandleLogMessage(TestMessageLevel level, string message)
EqtTrace.Warning(message);
break;
case TestMessageLevel.Error:
this.Errors.Add(message);
EqtTrace.Error(message);
break;
default:
Expand Down
1 change: 1 addition & 0 deletions test/E2ETests/Smoke.E2E.Tests/Smoke.E2E.Tests.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
<Compile Include="AssertExtensibilityTests.cs" />
<Compile Include="TimeoutTests.cs" />
<Compile Include="CompatTests.cs" />
<Compile Include="TestProjectFSharpTests.cs" />
<Compile Include="CustomTestExecutionExtensibilityTests.cs" />
<Compile Include="DataSourceTests.cs" />
<Compile Include="DeploymentTests.cs" />
Expand Down
23 changes: 23 additions & 0 deletions test/E2ETests/Smoke.E2E.Tests/TestProjectFSharpTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT license. See LICENSE file in the project root for full license information.

namespace MSTestAdapter.Smoke.E2ETests
{
using Microsoft.MSTestV2.CLIAutomation;
using Microsoft.VisualStudio.TestTools.UnitTesting;

[TestClass]
public class TestProjectFSharpTests : CLITestBase
{
private const string TestAssembly = "FSharpTestProject.dll";

[TestMethod]
public void ExecuteCustomTestExtensibilityTests()
{
this.InvokeVsTestForExecution(new string[] { TestAssembly });
this.ValidateFailedTestsCount(0);
this.ValidatePassedTestsCount(1);
this.ValidatePassedTestsContain("Test method passing with a . in it");
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TestFxRoot Condition="$(TestFxRoot) == ''">..\..\..\..\</TestFxRoot>
</PropertyGroup>

<PropertyGroup>
<TargetFramework>net452</TargetFramework>
<IsPackable>false</IsPackable>
<GenerateProgramFile>false</GenerateProgramFile>
<AppendTargetFrameworkToOutputPath>false</AppendTargetFrameworkToOutputPath>
<OutputPath>$(TestFxRoot)artifacts\TestAssets\</OutputPath>
</PropertyGroup>

<ItemGroup>
<ProjectReference Include="..\..\..\..\src\TestFramework\Extension.Desktop\Extension.Desktop.csproj" />
<ProjectReference Include="..\..\..\..\src\TestFramework\MSTest.Core\MSTest.Core.csproj" />
</ItemGroup>

<ItemGroup>
<Compile Include="Tests.fs" />
</ItemGroup>

</Project>
11 changes: 11 additions & 0 deletions test/E2ETests/TestAssets/FSharpTestProject/Tests.fs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
namespace FSharpTestProject

open System
open Microsoft.VisualStudio.TestTools.UnitTesting

[<TestClass>]
type UnitTest1 () =

[<TestMethod>]
member this.``Test method passing with a . in it`` () =
Assert.IsTrue(true);
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ public class TestCaseExtensionsTests
[TestMethod]
public void ToUnitTestElementShouldReturnUnitTestElementWithFieldsSet()
{
TestCase testCase = new TestCase("DummyClass.DummyMethod", new Uri("DummyUri", UriKind.Relative), Assembly.GetCallingAssembly().FullName);
TestCase testCase = new TestCase("DummyClassName.DummyMethod", new Uri("DummyUri", UriKind.Relative), Assembly.GetCallingAssembly().FullName);
testCase.DisplayName = "DummyDisplayName";
var testCategories = new[] { "DummyCategory" };

Expand Down