Bug 1872171 - add and support --restartAfterFailure for desktop mochi…

…tests. r=ahal

Differential Revision: https://phabricator.services.mozilla.com/D197365

testing/mochitest/mochitest_options.py

@@ -937,6 +937,14 @@ class MochitestArguments(ArgumentContainer):
                 "help": "Compare preferences at the end of each test and report changed ones as failures.",
             },
         ],
+        [
+            ["--restart-after-failure"],
+            {
+                "dest": "restartAfterFailure",
+                "default": False,
+                "help": "Terminate the session on first failure and restart where you left off.",
+            },
+        ],
     ]
 
     defaults = {

testing/mochitest/runtests.py

@@ -2736,6 +2736,7 @@ def runApp(
         detectShutdownLeaks=False,
         screenshotOnFail=False,
         bisectChunk=None,
+        restartAfterFailure=False,
         marionette_args=None,
         e10s=True,
         runFailures=False,
@@ -2843,6 +2844,7 @@ def runApp(
                 shutdownLeaks=shutdownLeaks,
                 lsanLeaks=lsanLeaks,
                 bisectChunk=bisectChunk,
+                restartAfterFailure=restartAfterFailure,
             )
 
             def timeoutHandler():
@@ -3162,6 +3164,20 @@ def runMochitests(self, options, testsToRun, manifestToFilter=None):
 
             if options.bisectChunk:
                 status = bisect.post_test(options, self.expectedError, self.result)
+            elif options.restartAfterFailure:
+                # NOTE: ideally browser will halt on first failure, then this will always be the last test
+                if not self.expectedError:
+                    status = -1
+                else:
+                    firstFail = len(testsToRun)
+                    for key in self.expectedError:
+                        full_key = [x for x in testsToRun if key in x]
+                        if full_key:
+                            if testsToRun.index(full_key[0]) < firstFail:
+                                firstFail = testsToRun.index(full_key[0])
+                    testsToRun = testsToRun[firstFail + 1 :]
+                    if testsToRun == []:
+                        status = -1
             else:
                 status = -1
 
@@ -3757,6 +3773,7 @@ def doTests(self, options, testsToFilter=None, manifestToFilter=None):
                     detectShutdownLeaks=detectShutdownLeaks,
                     screenshotOnFail=options.screenshotOnFail,
                     bisectChunk=options.bisectChunk,
+                    restartAfterFailure=options.restartAfterFailure,
                     marionette_args=marionette_args,
                     e10s=options.e10s,
                     runFailures=options.runFailures,
@@ -3920,6 +3937,7 @@ def __init__(
             shutdownLeaks=None,
             lsanLeaks=None,
             bisectChunk=None,
+            restartAfterFailure=None,
         ):
             """
             harness -- harness instance
@@ -3933,6 +3951,7 @@ def __init__(
             self.shutdownLeaks = shutdownLeaks
             self.lsanLeaks = lsanLeaks
             self.bisectChunk = bisectChunk
+            self.restartAfterFailure = restartAfterFailure
             self.browserProcessId = None
             self.stackFixerFunction = self.stackFixer()
 
@@ -3963,7 +3982,7 @@ def outputHandlers(self):
                 self.trackLSANLeaks,
                 self.countline,
             ]
-            if self.bisectChunk:
+            if self.bisectChunk or self.restartAfterFailure:
                 handlers.append(self.record_result)
                 handlers.append(self.first_error)
 
@@ -4148,6 +4167,11 @@ def run_test_harness(parser, options):
     if options.flavor in ("plain", "a11y", "browser", "chrome"):
         options.runByManifest = True
 
+    # run until failure, then loop until all tests have ran
+    # using looping similar to bisection code
+    if options.restartAfterFailure:
+        options.runUntilFailure = True
+
     if options.verify or options.verify_fission:
         result = runner.verifyTests(options)
     else:

testing/mochitest/runtestsremote.py

@@ -337,6 +337,7 @@ def runApp(
         detectShutdownLeaks=False,
         screenshotOnFail=False,
         bisectChunk=None,
+        restartAfterFailure=False,
         marionette_args=None,
         e10s=True,
         runFailures=False,

testing/mochitest/tests/SimpleTest/TestRunner.js

@@ -218,7 +218,10 @@ TestRunner._checkForHangs = function () {
 
       // If we have too many timeouts, give up. We don't want to wait hours
       // for results if some bug causes lots of tests to time out.
-      if (++TestRunner._numTimeouts >= TestRunner.maxTimeouts) {
+      if (
+        ++TestRunner._numTimeouts >= TestRunner.maxTimeouts ||
+        TestRunner.runUntilFailure
+      ) {
         TestRunner._haltTests = true;
 
         TestRunner.currentTestURL = "(SimpleTest/TestRunner.js)";

testing/mochitest/tests/python/conftest.py

@@ -52,6 +52,10 @@ def runtests(setup_test_harness, binary, parser, request):
     if "runFailures" in request.fixturenames:
         runFailures = request.getfixturevalue("runFailures")
 
+    restartAfterFailure = False
+    if "restartAfterFailure" in request.fixturenames:
+        restartAfterFailure = request.getfixturevalue("restartAfterFailure")
+
     setup_test_harness(*setup_args, flavor=flavor)
 
     runtests = pytest.importorskip("runtests")
@@ -74,6 +78,7 @@ def runtests(setup_test_harness, binary, parser, request):
             "app": binary,
             "flavor": flavor,
             "runFailures": runFailures,
+            "restartAfterFailure": restartAfterFailure,
             "keep_open": False,
             "log_raw": [buf],
         }
@@ -99,15 +104,20 @@ def runtests(setup_test_harness, binary, parser, request):
     options.update(getattr(request.module, "OPTIONS", {}))
 
     def normalize(test):
-        return {
-            "name": test,
-            "relpath": test,
-            "path": os.path.join(test_root, test),
-            # add a dummy manifest file because mochitest expects it
-            "manifest": os.path.join(test_root, manifest_name),
-            "manifest_relpath": manifest_name,
-            "skip-if": runFailures,
-        }
+        if isinstance(test, str):
+            test = [test]
+        return [
+            {
+                "name": t,
+                "relpath": t,
+                "path": os.path.join(test_root, t),
+                # add a dummy manifest file because mochitest expects it
+                "manifest": os.path.join(test_root, manifest_name),
+                "manifest_relpath": manifest_name,
+                "skip-if": runFailures,
+            }
+            for t in test
+        ]
 
     def inner(*tests, **opts):
         assert len(tests) > 0
@@ -118,7 +128,7 @@ def inner(*tests, **opts):
             manifest = TestManifest()
             options["manifestFile"] = manifest
             # pylint --py3k: W1636
-            manifest.tests.extend(list(map(normalize, tests)))
+            manifest.tests.extend(list(map(normalize, tests))[0])
             options.update(opts)
 
         result = runtests.run_test_harness(parser, Namespace(**options))

testing/mochitest/tests/python/files/browser_fail2.js

@@ -0,0 +1,3 @@
+function test() {
+  ok(false, "Test is ok");
+}

testing/mochitest/tests/python/files/test_fail2.html

@@ -0,0 +1,24 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1343659
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test Fail</title>
+  <script src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+    ok(false, "Test is ok");
+  </script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1343659">Mozilla Bug 1343659</a>
+<p id="display"></p>
+<div id="content" style="display: none">
+
+</div>
+<pre id="test">
+</pre>
+</body>
+</html>

testing/mochitest/tests/python/test_mochitest_integration.py

@@ -139,6 +139,41 @@ def test_output_fail(flavor, runFailures, runtests, test_name):
     assert lines[0]["status"] == results["line_status"]
 
 
+@pytest.mark.parametrize("runFailures", ["selftest", ""])
+@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])
+def test_output_restart_after_failure(flavor, runFailures, runtests, test_name):
+    extra_opts = {}
+    results = {
+        "status": 0 if runFailures else 1,
+        "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING,
+        "log_level": (INFO, WARNING),
+        "lines": 2,
+        "line_status": "PASS" if runFailures else "FAIL",
+    }
+    extra_opts["restartAfterFailure"] = True
+    if runFailures:
+        extra_opts["runFailures"] = runFailures
+        extra_opts["crashAsPass"] = True
+        extra_opts["timeoutAsPass"] = True
+
+    tests = [test_name("fail"), test_name("fail2")]
+    status, lines = runtests(tests, **extra_opts)
+    assert status == results["status"]
+
+    tbpl_status, log_level, summary = get_mozharness_status(lines, status)
+    assert tbpl_status == results["tbpl_status"]
+    assert log_level in results["log_level"]
+
+    # Ensure the harness cycled when failing (look for launching browser)
+    start_lines = [
+        line for line in lines if "Application command:" in line.get("message", "")
+    ]
+    if not runFailures:
+        assert len(start_lines) == results["lines"]
+    else:
+        assert len(start_lines) == 1
+
+
 @pytest.mark.skip_mozinfo("!crashreporter")
 @pytest.mark.parametrize("runFailures", ["selftest", ""])
 @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])