From a8c9713ea38d01dbfa19ed1a2499fb93f90d2101 Mon Sep 17 00:00:00 2001 From: Joel Maher Date: Fri, 5 Jan 2024 21:52:36 +0000 Subject: [PATCH] Bug 1872171 - add and support --restartAfterFailure for desktop mochitests. r=ahal Differential Revision: https://phabricator.services.mozilla.com/D197365 --- testing/mochitest/mochitest_options.py | 8 +++++ testing/mochitest/runtests.py | 26 +++++++++++++- testing/mochitest/runtestsremote.py | 1 + .../mochitest/tests/SimpleTest/TestRunner.js | 5 ++- testing/mochitest/tests/python/conftest.py | 30 ++++++++++------ .../tests/python/files/browser_fail2.js | 3 ++ .../tests/python/files/test_fail2.html | 24 +++++++++++++ .../python/test_mochitest_integration.py | 35 +++++++++++++++++++ 8 files changed, 120 insertions(+), 12 deletions(-) create mode 100644 testing/mochitest/tests/python/files/browser_fail2.js create mode 100644 testing/mochitest/tests/python/files/test_fail2.html diff --git a/testing/mochitest/mochitest_options.py b/testing/mochitest/mochitest_options.py index 3310bda7f863d..0b4a03f5b26f3 100644 --- a/testing/mochitest/mochitest_options.py +++ b/testing/mochitest/mochitest_options.py @@ -937,6 +937,14 @@ class MochitestArguments(ArgumentContainer): "help": "Compare preferences at the end of each test and report changed ones as failures.", }, ], + [ + ["--restart-after-failure"], + { + "dest": "restartAfterFailure", + "default": False, + "help": "Terminate the session on first failure and restart where you left off.", + }, + ], ] defaults = { diff --git a/testing/mochitest/runtests.py b/testing/mochitest/runtests.py index cb82ac0f7b45e..d058bdb5c3826 100644 --- a/testing/mochitest/runtests.py +++ b/testing/mochitest/runtests.py @@ -2736,6 +2736,7 @@ def runApp( detectShutdownLeaks=False, screenshotOnFail=False, bisectChunk=None, + restartAfterFailure=False, marionette_args=None, e10s=True, runFailures=False, @@ -2843,6 +2844,7 @@ def runApp( shutdownLeaks=shutdownLeaks, lsanLeaks=lsanLeaks, bisectChunk=bisectChunk, + restartAfterFailure=restartAfterFailure, ) def timeoutHandler(): @@ -3162,6 +3164,20 @@ def runMochitests(self, options, testsToRun, manifestToFilter=None): if options.bisectChunk: status = bisect.post_test(options, self.expectedError, self.result) + elif options.restartAfterFailure: + # NOTE: ideally browser will halt on first failure, then this will always be the last test + if not self.expectedError: + status = -1 + else: + firstFail = len(testsToRun) + for key in self.expectedError: + full_key = [x for x in testsToRun if key in x] + if full_key: + if testsToRun.index(full_key[0]) < firstFail: + firstFail = testsToRun.index(full_key[0]) + testsToRun = testsToRun[firstFail + 1 :] + if testsToRun == []: + status = -1 else: status = -1 @@ -3757,6 +3773,7 @@ def doTests(self, options, testsToFilter=None, manifestToFilter=None): detectShutdownLeaks=detectShutdownLeaks, screenshotOnFail=options.screenshotOnFail, bisectChunk=options.bisectChunk, + restartAfterFailure=options.restartAfterFailure, marionette_args=marionette_args, e10s=options.e10s, runFailures=options.runFailures, @@ -3920,6 +3937,7 @@ def __init__( shutdownLeaks=None, lsanLeaks=None, bisectChunk=None, + restartAfterFailure=None, ): """ harness -- harness instance @@ -3933,6 +3951,7 @@ def __init__( self.shutdownLeaks = shutdownLeaks self.lsanLeaks = lsanLeaks self.bisectChunk = bisectChunk + self.restartAfterFailure = restartAfterFailure self.browserProcessId = None self.stackFixerFunction = self.stackFixer() @@ -3963,7 +3982,7 @@ def outputHandlers(self): self.trackLSANLeaks, self.countline, ] - if self.bisectChunk: + if self.bisectChunk or self.restartAfterFailure: handlers.append(self.record_result) handlers.append(self.first_error) @@ -4148,6 +4167,11 @@ def run_test_harness(parser, options): if options.flavor in ("plain", "a11y", "browser", "chrome"): options.runByManifest = True + # run until failure, then loop until all tests have ran + # using looping similar to bisection code + if options.restartAfterFailure: + options.runUntilFailure = True + if options.verify or options.verify_fission: result = runner.verifyTests(options) else: diff --git a/testing/mochitest/runtestsremote.py b/testing/mochitest/runtestsremote.py index 0cc4cc02c892b..403cda9e95151 100644 --- a/testing/mochitest/runtestsremote.py +++ b/testing/mochitest/runtestsremote.py @@ -337,6 +337,7 @@ def runApp( detectShutdownLeaks=False, screenshotOnFail=False, bisectChunk=None, + restartAfterFailure=False, marionette_args=None, e10s=True, runFailures=False, diff --git a/testing/mochitest/tests/SimpleTest/TestRunner.js b/testing/mochitest/tests/SimpleTest/TestRunner.js index 91039491699a6..5f305a176bef4 100644 --- a/testing/mochitest/tests/SimpleTest/TestRunner.js +++ b/testing/mochitest/tests/SimpleTest/TestRunner.js @@ -218,7 +218,10 @@ TestRunner._checkForHangs = function () { // If we have too many timeouts, give up. We don't want to wait hours // for results if some bug causes lots of tests to time out. - if (++TestRunner._numTimeouts >= TestRunner.maxTimeouts) { + if ( + ++TestRunner._numTimeouts >= TestRunner.maxTimeouts || + TestRunner.runUntilFailure + ) { TestRunner._haltTests = true; TestRunner.currentTestURL = "(SimpleTest/TestRunner.js)"; diff --git a/testing/mochitest/tests/python/conftest.py b/testing/mochitest/tests/python/conftest.py index e418dfb8164ff..308b9d99cbfd0 100644 --- a/testing/mochitest/tests/python/conftest.py +++ b/testing/mochitest/tests/python/conftest.py @@ -52,6 +52,10 @@ def runtests(setup_test_harness, binary, parser, request): if "runFailures" in request.fixturenames: runFailures = request.getfixturevalue("runFailures") + restartAfterFailure = False + if "restartAfterFailure" in request.fixturenames: + restartAfterFailure = request.getfixturevalue("restartAfterFailure") + setup_test_harness(*setup_args, flavor=flavor) runtests = pytest.importorskip("runtests") @@ -74,6 +78,7 @@ def runtests(setup_test_harness, binary, parser, request): "app": binary, "flavor": flavor, "runFailures": runFailures, + "restartAfterFailure": restartAfterFailure, "keep_open": False, "log_raw": [buf], } @@ -99,15 +104,20 @@ def runtests(setup_test_harness, binary, parser, request): options.update(getattr(request.module, "OPTIONS", {})) def normalize(test): - return { - "name": test, - "relpath": test, - "path": os.path.join(test_root, test), - # add a dummy manifest file because mochitest expects it - "manifest": os.path.join(test_root, manifest_name), - "manifest_relpath": manifest_name, - "skip-if": runFailures, - } + if isinstance(test, str): + test = [test] + return [ + { + "name": t, + "relpath": t, + "path": os.path.join(test_root, t), + # add a dummy manifest file because mochitest expects it + "manifest": os.path.join(test_root, manifest_name), + "manifest_relpath": manifest_name, + "skip-if": runFailures, + } + for t in test + ] def inner(*tests, **opts): assert len(tests) > 0 @@ -118,7 +128,7 @@ def inner(*tests, **opts): manifest = TestManifest() options["manifestFile"] = manifest # pylint --py3k: W1636 - manifest.tests.extend(list(map(normalize, tests))) + manifest.tests.extend(list(map(normalize, tests))[0]) options.update(opts) result = runtests.run_test_harness(parser, Namespace(**options)) diff --git a/testing/mochitest/tests/python/files/browser_fail2.js b/testing/mochitest/tests/python/files/browser_fail2.js new file mode 100644 index 0000000000000..abcb6dae60d92 --- /dev/null +++ b/testing/mochitest/tests/python/files/browser_fail2.js @@ -0,0 +1,3 @@ +function test() { + ok(false, "Test is ok"); +} diff --git a/testing/mochitest/tests/python/files/test_fail2.html b/testing/mochitest/tests/python/files/test_fail2.html new file mode 100644 index 0000000000000..3d0555a5a0672 --- /dev/null +++ b/testing/mochitest/tests/python/files/test_fail2.html @@ -0,0 +1,24 @@ + + + + + + Test Fail + + + + + +Mozilla Bug 1343659 +

+ +
+
+ + diff --git a/testing/mochitest/tests/python/test_mochitest_integration.py b/testing/mochitest/tests/python/test_mochitest_integration.py index d252f7bfc383f..f881ea3f3064e 100644 --- a/testing/mochitest/tests/python/test_mochitest_integration.py +++ b/testing/mochitest/tests/python/test_mochitest_integration.py @@ -139,6 +139,41 @@ def test_output_fail(flavor, runFailures, runtests, test_name): assert lines[0]["status"] == results["line_status"] +@pytest.mark.parametrize("runFailures", ["selftest", ""]) +@pytest.mark.parametrize("flavor", ["plain", "browser-chrome"]) +def test_output_restart_after_failure(flavor, runFailures, runtests, test_name): + extra_opts = {} + results = { + "status": 0 if runFailures else 1, + "tbpl_status": TBPL_SUCCESS if runFailures else TBPL_WARNING, + "log_level": (INFO, WARNING), + "lines": 2, + "line_status": "PASS" if runFailures else "FAIL", + } + extra_opts["restartAfterFailure"] = True + if runFailures: + extra_opts["runFailures"] = runFailures + extra_opts["crashAsPass"] = True + extra_opts["timeoutAsPass"] = True + + tests = [test_name("fail"), test_name("fail2")] + status, lines = runtests(tests, **extra_opts) + assert status == results["status"] + + tbpl_status, log_level, summary = get_mozharness_status(lines, status) + assert tbpl_status == results["tbpl_status"] + assert log_level in results["log_level"] + + # Ensure the harness cycled when failing (look for launching browser) + start_lines = [ + line for line in lines if "Application command:" in line.get("message", "") + ] + if not runFailures: + assert len(start_lines) == results["lines"] + else: + assert len(start_lines) == 1 + + @pytest.mark.skip_mozinfo("!crashreporter") @pytest.mark.parametrize("runFailures", ["selftest", ""]) @pytest.mark.parametrize("flavor", ["plain", "browser-chrome"])