Skip to content

Commit

Permalink
Output Ignore Message for Ignored Tests (#165)
Browse files Browse the repository at this point in the history 
* Outpt Ignore Message for Ignored Tests

* PR comments
  • Loading branch information
@jayaranigarg
jayaranigarg committed Apr 26, 2017
1 parent 69e6153 commit bd95f57
Show file tree
Hide file tree
Showing 12 changed files with 220 additions and 145 deletions.
3 changes: 0 additions & 3 deletions src/Adapter/MSTest.CoreAdapter/Constants.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,8 +30,6 @@ internal static class Constants

#region Test Property registration

internal static readonly TestProperty TestEnabledProperty = TestProperty.Register("MSTestDiscovererv2.IsEnabled", IsEnabledLabel, typeof(bool), TestPropertyAttributes.Hidden, typeof(TestCase));

internal static readonly TestProperty TestClassNameProperty = TestProperty.Register("MSTestDiscovererv2.TestClassName", TestClassNameLabel, typeof(string), TestPropertyAttributes.Hidden, typeof(TestCase));

internal static readonly TestProperty AsyncTestProperty = TestProperty.Register("MSTestDiscovererv2.IsAsync", IsAsyncLabel, typeof(bool), TestPropertyAttributes.Hidden, typeof(TestCase));
Expand All @@ -53,7 +51,6 @@ internal static class Constants
/// to attributes on tests, and may be available in command line/TeamBuild to filter tests.
/// These Property names should not be localized.
/// </summary>
private const string IsEnabledLabel = "IsEnabled";
private const string TestClassNameLabel = "ClassName";
private const string IsAsyncLabel = "IsAsync";
private const string TestCategoryLabel = "TestCategory";
Expand Down
24 changes: 0 additions & 24 deletions src/Adapter/MSTest.CoreAdapter/Discovery/TypeEnumerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,6 @@ internal class TypeEnumerator
private readonly TestMethodValidator testMethodValidator;
private readonly ReflectHelper reflectHelper;

/// <summary>
/// Whether test class has [Ignore] attribute on it. Nullable so that we can cache the value.
/// </summary>
private bool? isIgnoreAttributeOnClass;

/// <summary>
/// Initializes a new instance of the <see cref="TypeEnumerator"/> class.
/// </summary>
Expand All @@ -49,23 +44,6 @@ internal TypeEnumerator(Type type, string assemblyName, ReflectHelper reflectHel
this.testMethodValidator = testMethodValidator;
}

/// <summary>
/// Gets a value indicating whether the test class has an ignored attribute or not.
/// </summary>
internal bool IsIgnoreAttributeOnTestClass
{
get
{
if (this.isIgnoreAttributeOnClass == null)
{
this.isIgnoreAttributeOnClass = this.reflectHelper.IsAttributeDefined(this.type, typeof(IgnoreAttribute), false);
}

Debug.Assert(this.isIgnoreAttributeOnClass.HasValue, "isIgnoreAttributeOnClass.HasValue");
return this.isIgnoreAttributeOnClass.Value;
}
}

/// <summary>
/// Walk through all methods in the type, and find out the test methods
/// </summary>
Expand Down Expand Up @@ -139,8 +117,6 @@ internal UnitTestElement GetTestFromMethod(MethodInfo method, ICollection<string
var asyncTypeName = method.GetAsyncTypeName();
testElement.AsyncTypeName = asyncTypeName;

testElement.Ignored = this.IsIgnoreAttributeOnTestClass || this.reflectHelper.IsAttributeDefined(method, typeof(IgnoreAttribute), false);

testElement.TestCategory = this.reflectHelper.GetCategories(method);

var traits = this.reflectHelper.GetTestPropertiesAsTraits(method);
Expand Down
25 changes: 9 additions & 16 deletions src/Adapter/MSTest.CoreAdapter/Execution/TestExecutionManager.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ public void RunTests(IEnumerable<string> sources, IRunContext runContext, IFrame
internal virtual void ExecuteTests(IEnumerable<TestCase> tests, IRunContext runContext, IFrameworkHandle frameworkHandle, bool isDeploymentDone)
{
var testsBySource = from test in tests
group test by test.Source into testGroup
select new { Source = testGroup.Key, Tests = testGroup };
group test by test.Source into testGroup
select new { Source = testGroup.Key, Tests = testGroup };

foreach (var group in testsBySource)
{
Expand Down Expand Up @@ -263,23 +263,16 @@ private void ExecuteTestsWithTestRunner(
"Executing test {0}",
unitTestElement.TestMethod.Name);

if (unitTestElement.Ignored)
// this is done so that appropriate values of testcontext properties are set at source level
// and are merged with session level parameters
var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source);

if (this.sessionParameters != null && this.sessionParameters.Count > 0)
{
unitTestResult = new[] { new UnitTestResult(UnitTestOutcome.Ignored, null) };
sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value);
}
else
{
// this is done so that appropriate values of testcontext properties are set at source level
// and are merged with session level parameters
var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source);

if (this.sessionParameters != null && this.sessionParameters.Count > 0)
{
sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value);
}

unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, sourceLevelParameters);
}
unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, sourceLevelParameters);

PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executed test {0}",
Expand Down
66 changes: 65 additions & 1 deletion src/Adapter/MSTest.CoreAdapter/Execution/TestMethodRunner.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,11 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.Globalization;
using System.Reflection;

using Extensions;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Helpers;
using Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.ObjectModel;
using Microsoft.VisualStudio.TestPlatform.MSTestAdapter.PlatformServices.Interface;

Expand All @@ -18,7 +20,7 @@ namespace Microsoft.VisualStudio.TestPlatform.MSTest.TestAdapter.Execution
/// <summary>
/// This class is responsible to running tests and converting framework TestResults to adapter TestResults.
/// </summary>
public class TestMethodRunner
internal class TestMethodRunner
{
/// <summary>
/// Test context which needs to be passed to the various methods of the test
Expand All @@ -40,6 +42,11 @@ public class TestMethodRunner
/// </summary>
private readonly bool captureDebugTraces;

/// <summary>
/// Helper for reflection API's.
/// </summary>
private ReflectHelper reflectHelper;

/// <summary>
/// Initializes a new instance of the <see cref="TestMethodRunner"/> class.
/// </summary>
Expand All @@ -60,6 +67,7 @@ public TestMethodRunner(
TestMethod testMethod,
ITestContext testContext,
bool captureDebugTraces)
: this(testMethodInfo, testMethod, testContext, captureDebugTraces, new ReflectHelper())
{
Debug.Assert(testMethodInfo != null, "testMethodInfo should not be null");
Debug.Assert(testMethod != null, "testMethod should not be null");
Expand All @@ -71,6 +79,42 @@ public TestMethodRunner(
this.captureDebugTraces = captureDebugTraces;
}

/// <summary>
/// Initializes a new instance of the <see cref="TestMethodRunner"/> class.
/// </summary>
/// <param name="testMethodInfo">
/// The test method info.
/// </param>
/// <param name="testMethod">
/// The test method.
/// </param>
/// <param name="testContext">
/// The test context.
/// </param>
/// <param name="captureDebugTraces">
/// The capture debug traces.
/// </param>
/// <param name="reflectHelper">
/// The reflect Helper object.
/// </param>
public TestMethodRunner(
TestMethodInfo testMethodInfo,
TestMethod testMethod,
ITestContext testContext,
bool captureDebugTraces,
ReflectHelper reflectHelper)
{
Debug.Assert(testMethodInfo != null, "testMethodInfo should not be null");
Debug.Assert(testMethod != null, "testMethod should not be null");
Debug.Assert(testContext != null, "testContext should not be null");

this.testMethodInfo = testMethodInfo;
this.test = testMethod;
this.testContext = testContext;
this.captureDebugTraces = captureDebugTraces;
this.reflectHelper = reflectHelper;
}

/// <summary>
/// Executes a test
/// </summary>
Expand All @@ -84,6 +128,26 @@ internal UnitTestResult[] Execute()
string inittestContextMessages = string.Empty;

UnitTestResult[] result = null;

string ignoreMessage = null;
var isIgnoreAttributeOnClass = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.Parent.ClassType, typeof(UTF.IgnoreAttribute), false);
var isIgnoreAttributeOnMethod = this.reflectHelper.IsAttributeDefined(this.testMethodInfo.TestMethod, typeof(UTF.IgnoreAttribute), false);

if (isIgnoreAttributeOnClass)
{
ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.Parent.ClassType.GetTypeInfo());
}

if (string.IsNullOrEmpty(ignoreMessage) && isIgnoreAttributeOnMethod)
{
ignoreMessage = this.reflectHelper.GetIgnoreMessage(this.testMethodInfo.TestMethod);
}

if (isIgnoreAttributeOnClass || isIgnoreAttributeOnMethod)
{
return new[] { new UnitTestResult(UnitTestOutcome.Ignored, ignoreMessage) };
}

try
{
using (LogMessageListener logListener = new LogMessageListener(this.captureDebugTraces))
Expand Down
2 changes: 0 additions & 2 deletions src/Adapter/MSTest.CoreAdapter/Extensions/TestCaseExtensions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,14 +21,12 @@ internal static class TestCaseExtensions
internal static UnitTestElement ToUnitTestElement(this TestCase testCase, string source)
{
var isAsync = (testCase.GetPropertyValue(Constants.AsyncTestProperty) as bool?) ?? false;
var isEnabled = (testCase.GetPropertyValue(Constants.TestEnabledProperty) as bool?) ?? true;
var testClassName = testCase.GetPropertyValue(Constants.TestClassNameProperty) as string;

TestMethod testMethod = new TestMethod(testCase.DisplayName, testClassName, source, isAsync);

UnitTestElement testElement = new UnitTestElement(testMethod)
{
Ignored = !isEnabled,
IsAsync = isAsync,
TestCategory = testCase.GetPropertyValue(Constants.TestCategoryProperty) as string[],
Priority = testCase.GetPropertyValue(Constants.PriorityProperty) as int?
Expand Down
18 changes: 18 additions & 0 deletions src/Adapter/MSTest.CoreAdapter/Helpers/ReflectHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,24 @@ internal virtual Trait GetTestPriorityAsTraits(int? testPriority)
return (priorityAttribute[0] as PriorityAttribute).Priority;
}

/// <summary>
/// Priority if any set for test method. Will return priority if attribute is applied to TestMethod
/// else null;
/// </summary>
/// <param name="ignoreAttributeProvider">The member to inspect.</param>
/// <returns>Priority value if defined. Null otherwise.</returns>
internal virtual string GetIgnoreMessage(MemberInfo ignoreAttributeProvider)
{
var ignoreAttribute = GetCustomAttributes(ignoreAttributeProvider, typeof(IgnoreAttribute), true);

if (!ignoreAttribute.Any())
{
return null;
}

return (ignoreAttribute?.FirstOrDefault() as IgnoreAttribute).IgnoreMessage;
}

/// <summary>
/// KeyValue pairs that are provided by TestPropertyAttributes of the given test method.
/// </summary>
Expand Down
1 change: 0 additions & 1 deletion src/Adapter/MSTest.CoreAdapter/ObjectModel/UnitTestElement.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,6 @@ internal TestCase ToTestCase()
TestCase testCase = new TestCase(fullName, TestAdapter.Constants.ExecutorUri, this.TestMethod.AssemblyName);
testCase.DisplayName = this.TestMethod.Name;

testCase.SetPropertyValue(TestAdapter.Constants.TestEnabledProperty, !this.Ignored);
testCase.SetPropertyValue(TestAdapter.Constants.TestClassNameProperty, this.TestMethod.FullClassName);

// Many of the tests will not be async, so there is no point in sending extra data
Expand Down
23 changes: 23 additions & 0 deletions src/TestFramework/MSTest.Core/Attributes/VSTestAttributes.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,29 @@ public sealed class TestCleanupAttribute : Attribute
[AttributeUsage(AttributeTargets.Class | AttributeTargets.Method, AllowMultiple = false)]
public sealed class IgnoreAttribute : Attribute
{
/// <summary>
/// Initializes a new instance of the <see cref="IgnoreAttribute"/> class.
/// </summary>
public IgnoreAttribute()
: this(string.Empty)
{
}

/// <summary>
/// Initializes a new instance of the <see cref="IgnoreAttribute"/> class.
/// </summary>
/// <param name="message">
/// Message specifies reason for ignoring.
/// </param>
public IgnoreAttribute(string message)
{
this.IgnoreMessage = message;
}

/// <summary>
/// Gets the owner.
/// </summary>
public string IgnoreMessage { get; private set; }
}

/// <summary>
Expand Down
81 changes: 0 additions & 81 deletions test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Discovery/TypeEnumeratorTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,44 +219,6 @@ public void GetTestFromMethodShouldInitiateAsyncTypeNameCorrectly()
Assert.AreEqual(expectedAsyncTaskName, testElement.AsyncTypeName);
}

[TestMethod]
public void GetTestFromMethodShouldSetIgnoredPropertyToTrueIfSetOnTestClass()
{
this.SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameType: true);
TypeEnumerator typeEnumerator = this.GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName");
var methodInfo = typeof(DummyTestClass).GetMethod("MethodWithVoidReturnType");

// Setup mocks
this.mockReflectHelper.Setup(
rh => rh.IsAttributeDefined(typeof(DummyTestClass), typeof(UTF.IgnoreAttribute), false)).Returns(true);
this.mockReflectHelper.Setup(
rh => rh.IsAttributeDefined(methodInfo, typeof(UTF.IgnoreAttribute), false)).Returns(false);

var testElement = typeEnumerator.GetTestFromMethod(methodInfo, this.warnings);

Assert.IsNotNull(testElement);
Assert.IsTrue(testElement.Ignored);
}

[TestMethod]
public void GetTestFromMethodShouldSetIgnoredPropertyToTrueIfSetOnTestMethod()
{
this.SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameType: true);
TypeEnumerator typeEnumerator = this.GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName");
var methodInfo = typeof(DummyTestClass).GetMethod("MethodWithVoidReturnType");

// Setup mocks
this.mockReflectHelper.Setup(
rh => rh.IsAttributeDefined(typeof(DummyTestClass), typeof(UTF.IgnoreAttribute), false)).Returns(false);
this.mockReflectHelper.Setup(
rh => rh.IsAttributeDefined(methodInfo, typeof(UTF.IgnoreAttribute), false)).Returns(true);

var testElement = typeEnumerator.GetTestFromMethod(methodInfo, this.warnings);

Assert.IsNotNull(testElement);
Assert.IsTrue(testElement.Ignored);
}

[TestMethod]
public void GetTestFromMethodShouldSetIgnoredPropertyToFalseIfNotSetOnTestClassAndTestMethod()
{
Expand Down Expand Up @@ -406,49 +368,6 @@ public void GetTestFromMethodShouldSetDeploymentItems()

#endregion

#region IsIgnoreAttributeOnTestClass tests

[TestMethod]
public void IsIgnoreAttributeOnTestClassReturnsTrueIfIgnoreIsSetOnTestClass()
{
this.SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameType: true);
TypeEnumerator typeEnumerator = this.GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName");

// Setup mocks
this.mockReflectHelper.Setup(rh => rh.IsAttributeDefined(typeof(DummyTestClass), typeof(UTF.IgnoreAttribute), It.IsAny<bool>())).Returns(true);

Assert.IsTrue(typeEnumerator.IsIgnoreAttributeOnTestClass);
}

[TestMethod]
public void IsIgnoreAttributeOnTestClassReturnsFalseIfIgnoreIsNotSetOnTestClass()
{
this.SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameType: true);
TypeEnumerator typeEnumerator = this.GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName");

// Setup mocks
this.mockReflectHelper.Setup(rh => rh.IsAttributeDefined(typeof(DummyTestClass), typeof(UTF.IgnoreAttribute), It.IsAny<bool>())).Returns(false);

Assert.IsFalse(typeEnumerator.IsIgnoreAttributeOnTestClass);
}

[TestMethod]
public void IsIgnoreAttributeOnTestClassShouldBeCached()
{
this.SetupTestClassAndTestMethods(isValidTestClass: true, isValidTestMethod: true, isMethodFromSameType: true);
TypeEnumerator typeEnumerator = this.GetTypeEnumeratorInstance(typeof(DummyTestClass), "DummyAssemblyName");

// Setup mocks
this.mockReflectHelper.Setup(rh => rh.IsAttributeDefined(typeof(DummyTestClass), typeof(UTF.IgnoreAttribute), It.IsAny<bool>())).Returns(true);

Assert.IsTrue(typeEnumerator.IsIgnoreAttributeOnTestClass);

this.mockReflectHelper.Setup(rh => rh.IsAttributeDefined(typeof(DummyTestClass), typeof(UTF.IgnoreAttribute), It.IsAny<bool>())).Returns(false);
Assert.IsTrue(typeEnumerator.IsIgnoreAttributeOnTestClass);
}

#endregion

#region private methods

private void SetupTestClassAndTestMethods(bool isValidTestClass, bool isValidTestMethod, bool isMethodFromSameType)
Expand Down
Loading

0 comments on commit bd95f57

Please sign in to comment.