Skip to content

Commit

Permalink
[3.11] bpo-46523: fix tests rerun when setUp[Class|Module] fails (G…
Browse files Browse the repository at this point in the history
…H-30895) (GH-103342)

(cherry picked from commit 9953860)

Co-authored-by: Nikita Sobolev <mail@sobolevn.me>
  • Loading branch information
ambv and sobolevn authored Apr 7, 2023
1 parent 8740fd8 commit ecb09a8
Show file tree
Hide file tree
Showing 3 changed files with 194 additions and 3 deletions.
36 changes: 34 additions & 2 deletions Lib/test/libregrtest/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,19 @@
# Must be smaller than buildbot "1200 seconds without output" limit.
EXIT_TIMEOUT = 120.0

# gh-90681: When rerunning tests, we might need to rerun the whole
# class or module suite if some its life-cycle hooks fail.
# Test level hooks are not affected.
_TEST_LIFECYCLE_HOOKS = frozenset((
'setUpClass', 'tearDownClass',
'setUpModule', 'tearDownModule',
))

EXITCODE_BAD_TEST = 2
EXITCODE_INTERRUPTED = 130
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4


class Regrtest:
"""Execute a test suite.
Expand Down Expand Up @@ -331,8 +344,12 @@ def rerun_failed_tests(self):

errors = result.errors or []
failures = result.failures or []
error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
error_names = [
self.normalize_test_name(test_full_name, is_error=True)
for (test_full_name, *_) in errors]
failure_names = [
self.normalize_test_name(test_full_name)
for (test_full_name, *_) in failures]
self.ns.verbose = True
orig_match_tests = self.ns.match_tests
if errors or failures:
Expand All @@ -358,6 +375,21 @@ def rerun_failed_tests(self):

self.display_result()

def normalize_test_name(self, test_full_name, *, is_error=False):
short_name = test_full_name.split(" ")[0]
if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
# This means that we have a failure in a life-cycle hook,
# we need to rerun the whole module or class suite.
# Basically the error looks like this:
# ERROR: setUpClass (test.test_reg_ex.RegTest)
# or
# ERROR: setUpModule (test.test_reg_ex)
# So, we need to parse the class / module name.
lpar = test_full_name.index('(')
rpar = test_full_name.index(')')
return test_full_name[lpar + 1: rpar].split('.')[-1]
return short_name

def display_result(self):
# If running the test suite for PGO then no one cares about results.
if self.ns.pgo:
Expand Down
2 changes: 1 addition & 1 deletion Lib/test/support/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1097,7 +1097,7 @@ def _run_suite(suite):
if junit_xml_list is not None:
junit_xml_list.append(result.get_xml_element())

if not result.testsRun and not result.skipped:
if not result.testsRun and not result.skipped and not result.errors:
raise TestDidNotRun
if not result.wasSuccessful():
if len(result.errors) == 1 and not result.failures:
Expand Down
159 changes: 159 additions & 0 deletions Lib/test/test_regrtest.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'

EXITCODE_BAD_TEST = 2
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4
EXITCODE_INTERRUPTED = 130

TEST_INTERRUPTED = textwrap.dedent("""
from signal import SIGINT, raise_signal
try:
Expand Down Expand Up @@ -1115,6 +1120,160 @@ def test_fail_once(self):
self.check_executed_tests(output, [testname],
rerun={testname: "test_fail_once"})

def test_rerun_setup_class_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
class ExampleTests(unittest.TestCase):
@classmethod
def setUpClass(self):
raise RuntimeError('Fail')
def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: "ExampleTests"})

def test_rerun_teardown_class_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
class ExampleTests(unittest.TestCase):
@classmethod
def tearDownClass(self):
raise RuntimeError('Fail')
def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: "ExampleTests"})

def test_rerun_setup_module_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
def setUpModule():
raise RuntimeError('Fail')
class ExampleTests(unittest.TestCase):
def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: testname})

def test_rerun_teardown_module_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
def tearDownModule():
raise RuntimeError('Fail')
class ExampleTests(unittest.TestCase):
def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: testname})

def test_rerun_setup_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
class ExampleTests(unittest.TestCase):
def setUp(self):
raise RuntimeError('Fail')
def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: "test_success"})

def test_rerun_teardown_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
class ExampleTests(unittest.TestCase):
def tearDown(self):
raise RuntimeError('Fail')
def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: "test_success"})

def test_rerun_async_setup_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
class ExampleTests(unittest.IsolatedAsyncioTestCase):
async def asyncSetUp(self):
raise RuntimeError('Fail')
async def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: "test_success"})

def test_rerun_async_teardown_hook_failure(self):
# FAILURE then FAILURE
code = textwrap.dedent("""
import unittest
class ExampleTests(unittest.IsolatedAsyncioTestCase):
async def asyncTearDown(self):
raise RuntimeError('Fail')
async def test_success(self):
return
""")
testname = self.create_test(code=code)

output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
self.check_executed_tests(output, testname,
failed=[testname],
rerun={testname: "test_success"})

def test_no_tests_ran(self):
code = textwrap.dedent("""
import unittest
Expand Down

0 comments on commit ecb09a8

Please sign in to comment.