TestTimeout in RunSettings (#403)

singhsarab · web-flow · commit 6d10ac9382a4 · 2018-04-18T18:35:11.000+05:30
* Adding global test timeout configurable via runsettings

* Adding negative test as per comments.
diff --git a/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs b/src/Adapter/MSTest.CoreAdapter/Execution/TypeCache.cs
@@ -286,7 +286,7 @@ private TestClassInfo CreateClassInfo(Type classType, TestMethod testMethod)
             foreach (var methodInfo in classType.GetTypeInfo().DeclaredMethods)
             {
                 // Update test initialize/cleanup method
-                this.UpdateInfoIfTestInitializeOrCleanupMethod(classInfo, methodInfo, isBase: false, instanceMethods: instanceMethods,  testInitializeAttributeType: testInitializeAttributeType, testCleanupAttributeType: testCleanupAttributeType);
+                this.UpdateInfoIfTestInitializeOrCleanupMethod(classInfo, methodInfo, isBase: false, instanceMethods: instanceMethods, testInitializeAttributeType: testInitializeAttributeType, testCleanupAttributeType: testCleanupAttributeType);
 
                 if (this.IsAssemblyOrClassInitializeMethod(methodInfo, classInitializeAttributeType))
                 {
@@ -634,6 +634,7 @@ private int GetTestTimeout(MethodInfo methodInfo, TestMethod testMethod)
         {
             Debug.Assert(methodInfo != null, "TestMethod should be non-null");
             var timeoutAttribute = this.reflectionHelper.GetAttribute<TimeoutAttribute>(methodInfo);
+            var globalTimeout = MSTestSettings.CurrentSettings.TestTimeout;
 
             if (timeoutAttribute != null)
             {
@@ -645,6 +646,10 @@ private int GetTestTimeout(MethodInfo methodInfo, TestMethod testMethod)
 
                 return timeoutAttribute.Timeout;
             }
+            else if (globalTimeout > 0)
+            {
+                return globalTimeout;
+            }
 
             return TestMethodInfo.TimeoutWhenNotSet;
         }
diff --git a/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs b/src/Adapter/MSTest.CoreAdapter/MSTestSettings.cs
@@ -55,6 +55,7 @@ public MSTestSettings()
             this.ForcedLegacyMode = false;
             this.TestSettingsFile = null;
             this.DisableParallelization = false;
+            this.TestTimeout = 0;
         }
 
         /// <summary>
@@ -143,6 +144,11 @@ private set
         /// </remarks>
         public bool DisableParallelization { get; private set; }
 
+        /// <summary>
+        ///  Gets specified global test case timeout
+        /// </summary>
+        public int TestTimeout { get; private set; }
+
         /// <summary>
         /// Populate settings based on existing settings object.
         /// </summary>
@@ -157,6 +163,7 @@ public static void PopulateSettings(MSTestSettings settings)
             CurrentSettings.ParallelizationWorkers = settings.ParallelizationWorkers;
             CurrentSettings.ParallelizationScope = settings.ParallelizationScope;
             CurrentSettings.DisableParallelization = settings.DisableParallelization;
+            CurrentSettings.TestTimeout = settings.TestTimeout;
         }
 
         /// <summary>
@@ -274,6 +281,7 @@ private static MSTestSettings ToSettings(XmlReader reader)
             //     <CaptureTraceOutput>true</CaptureTraceOutput>
             //     <MapInconclusiveToFailed>false</MapInconclusiveToFailed>
             //     <EnableBaseClassTestMethodsFromOtherAssemblies>false</EnableBaseClassTestMethodsFromOtherAssemblies>
+            //     <TestTimeout>5000<TestTimeout>
             //     <Parallelize>
             //        <Workers>4</Workers>
             //        <Scope>TestClass</Scope>
@@ -362,6 +370,16 @@ private static MSTestSettings ToSettings(XmlReader reader)
                                 break;
                             }
 
+                        case "TESTTIMEOUT":
+                            {
+                                if (int.TryParse(reader.ReadInnerXml(), out int testTimeout) && testTimeout > 0)
+                                {
+                                    settings.TestTimeout = testTimeout;
+                                }
+
+                                break;
+                            }
+
                         default:
                             {
                                 PlatformServiceProvider.Instance.SettingsProvider.Load(reader.ReadSubtree());
diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/Execution/TypeCacheTests.cs
@@ -54,6 +54,7 @@ public void TestInit()
         public void Cleanup()
         {
             PlatformServiceProvider.Instance = null;
+            MSTestSettings.Reset();
         }
 
         #region GetTestMethodInfo tests
@@ -815,6 +816,81 @@ public void GetTestMethodInfoShouldThrowWhenTimeoutIsIncorrect()
             Assert.AreEqual(expectedMessage, exception.Message);
         }
 
+        [TestMethodV1]
+        public void GetTestMethodInfoWhenTimeoutAttributeNotSetShouldReturnTestMethodInfoWithGlobalTimeout()
+        {
+            string runSettingxml =
+                @"<RunSettings>
+                    <MSTestV2>
+                        <TestTimeout>4000</TestTimeout>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings.PopulateSettings(MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias));
+
+            var type = typeof(DummyTestClassWithTestMethods);
+            var methodInfo = type.GetMethod("TestMethod");
+            var testMethod = new TestMethod(methodInfo.Name, type.FullName, "A", isAsync: false);
+
+            var testMethodInfo = this.typeCache.GetTestMethodInfo(
+                    testMethod,
+                    new TestContextImplementation(testMethod, null, new Dictionary<string, object>()),
+                    false);
+
+            Assert.AreEqual(4000, testMethodInfo.TestMethodOptions.Timeout);
+        }
+
+        [TestMethodV1]
+        public void GetTestMethodInfoWhenTimeoutAttributeSetShouldReturnTimeoutBasedOnAtrributeEvenIfGlobalTimeoutSet()
+        {
+            string runSettingxml =
+                @"<RunSettings>
+                    <MSTestV2>
+                        <TestTimeout>4000</TestTimeout>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings.PopulateSettings(MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias));
+
+            var type = typeof(DummyTestClassWithTestMethods);
+            var methodInfo = type.GetMethod("TestMethodWithTimeout");
+            var testMethod = new TestMethod(methodInfo.Name, type.FullName, "A", isAsync: false);
+
+            this.mockReflectHelper.Setup(rh => rh.IsAttributeDefined(methodInfo, typeof(UTF.TimeoutAttribute), false))
+               .Returns(true);
+
+            var testMethodInfo = this.typeCache.GetTestMethodInfo(
+                    testMethod,
+                    new TestContextImplementation(testMethod, null, new Dictionary<string, object>()),
+                    false);
+
+            Assert.AreEqual(10, testMethodInfo.TestMethodOptions.Timeout);
+        }
+
+        [TestMethodV1]
+        public void GetTestMethodInfoForInvalidGLobalTimeoutShouldReturnTestMethodInfoWithTimeoutZero()
+        {
+            string runSettingxml =
+                @"<RunSettings>
+                    <MSTestV2>
+                        <TestTimeout>30.5</TestTimeout>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings.PopulateSettings(MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias));
+
+            var type = typeof(DummyTestClassWithTestMethods);
+            var methodInfo = type.GetMethod("TestMethod");
+            var testMethod = new TestMethod(methodInfo.Name, type.FullName, "A", isAsync: false);
+
+            var testMethodInfo = this.typeCache.GetTestMethodInfo(
+                    testMethod,
+                    new TestContextImplementation(testMethod, null, new Dictionary<string, object>()),
+                    false);
+
+            Assert.AreEqual(0, testMethodInfo.TestMethodOptions.Timeout);
+        }
+
         [TestMethodV1]
         public void GetTestMethodInfoShouldReturnTestMethodInfoForMethodsAdornedWithADerivedTestMethodAttribute()
         {
diff --git a/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs b/test/UnitTests/MSTest.CoreAdapter.Unit.Tests/MSTestSettingsTests.cs
@@ -195,6 +195,35 @@ public void CaptureDebugTracesShouldBeConsumedFromRunSettingsWhenSpecified()
             Assert.AreEqual(adapterSettings.CaptureDebugTraces, false);
         }
 
+        [TestMethod]
+        public void TestTimeoutShouldBeConsumedFromRunSettingsWhenSpecified()
+        {
+            string runSettingxml =
+                @"<RunSettings>
+                    <MSTestV2>
+                        <TestTimeout>4000</TestTimeout>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias);
+
+            Assert.AreEqual(adapterSettings.TestTimeout, 4000);
+        }
+
+        [TestMethod]
+        public void TestTimeoutShouldBeSetToZeroIfNotSpecifiedInRunSettings()
+        {
+            string runSettingxml =
+                @"<RunSettings>
+                    <MSTestV2>
+                    </MSTestV2>
+                  </RunSettings>";
+
+            MSTestSettings adapterSettings = MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsNameAlias);
+
+            Assert.AreEqual(adapterSettings.TestTimeout, 0);
+        }
+
         [TestMethod]
         public void ParallelizationSettingsShouldNotBeSetByDefault()
         {
@@ -524,7 +553,7 @@ public void GetSettingsShouldOnlyPassTheElementSubTreeToPlatformService()
                         actualReader.Read();
                         observedxml = actualReader.ReadOuterXml();
                     }
-                 });
+                });
 
             MSTestSettings.GetSettings(runSettingxml, MSTestSettings.SettingsName);
             Assert.AreEqual(expectedrunSettingxml, observedxml);

Original file line number	Diff line number	Diff line change
`@@ -286,7 +286,7 @@ private TestClassInfo CreateClassInfo(Type classType, TestMethod testMethod)`
`286`	`286`	`foreach (var methodInfo in classType.GetTypeInfo().DeclaredMethods)`
`287`	`287`	`{`
`288`	`288`	`// Update test initialize/cleanup method`
`289`		`- this.UpdateInfoIfTestInitializeOrCleanupMethod(classInfo, methodInfo, isBase: false, instanceMethods: instanceMethods, testInitializeAttributeType: testInitializeAttributeType, testCleanupAttributeType: testCleanupAttributeType);`
	`289`	`+ this.UpdateInfoIfTestInitializeOrCleanupMethod(classInfo, methodInfo, isBase: false, instanceMethods: instanceMethods, testInitializeAttributeType: testInitializeAttributeType, testCleanupAttributeType: testCleanupAttributeType);`
`290`	`290`
`291`	`291`	`if (this.IsAssemblyOrClassInitializeMethod(methodInfo, classInitializeAttributeType))`
`292`	`292`	`{`
`@@ -634,6 +634,7 @@ private int GetTestTimeout(MethodInfo methodInfo, TestMethod testMethod)`
`634`	`634`	`{`
`635`	`635`	`Debug.Assert(methodInfo != null, "TestMethod should be non-null");`
`636`	`636`	`var timeoutAttribute = this.reflectionHelper.GetAttribute<TimeoutAttribute>(methodInfo);`
	`637`	`+ var globalTimeout = MSTestSettings.CurrentSettings.TestTimeout;`
`637`	`638`
`638`	`639`	`if (timeoutAttribute != null)`
`639`	`640`	`{`
`@@ -645,6 +646,10 @@ private int GetTestTimeout(MethodInfo methodInfo, TestMethod testMethod)`
`645`	`646`
`646`	`647`	`return timeoutAttribute.Timeout;`
`647`	`648`	`}`
	`649`	`+ else if (globalTimeout > 0)`
	`650`	`+ {`
	`651`	`+ return globalTimeout;`
	`652`	`+ }`
`648`	`653`
`649`	`654`	`return TestMethodInfo.TimeoutWhenNotSet;`
`650`	`655`	`}`