Skip to content

Commit 9953860

Browse files
sobolevnJelleZijlstraambv
authored
bpo-46523: fix tests rerun when setUp[Class|Module] fails (#30895)
Co-authored-by: Jelle Zijlstra <jelle.zijlstra@gmail.com> Co-authored-by: Łukasz Langa <lukasz@langa.pl>
1 parent 059bb04 commit 9953860

File tree

3 files changed

+184
-3
lines changed

3 files changed

+184
-3
lines changed

‎Lib/test/libregrtest/main.py‎

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,14 @@
2929
# Must be smaller than buildbot "1200 seconds without output" limit.
3030
EXIT_TIMEOUT = 120.0
3131

32+
# gh-90681: When rerunning tests, we might need to rerun the whole
33+
# class or module suite if some its life-cycle hooks fail.
34+
# Test level hooks are not affected.
35+
_TEST_LIFECYCLE_HOOKS = frozenset((
36+
'setUpClass', 'tearDownClass',
37+
'setUpModule', 'tearDownModule',
38+
))
39+
3240
EXITCODE_BAD_TEST = 2
3341
EXITCODE_INTERRUPTED = 130
3442
EXITCODE_ENV_CHANGED = 3
@@ -337,8 +345,12 @@ def rerun_failed_tests(self):
337345

338346
errors = result.errors or []
339347
failures = result.failures or []
340-
error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
341-
failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
348+
error_names = [
349+
self.normalize_test_name(test_full_name, is_error=True)
350+
for (test_full_name, *_) in errors]
351+
failure_names = [
352+
self.normalize_test_name(test_full_name)
353+
for (test_full_name, *_) in failures]
342354
self.ns.verbose = True
343355
orig_match_tests = self.ns.match_tests
344356
if errors or failures:
@@ -364,6 +376,21 @@ def rerun_failed_tests(self):
364376

365377
self.display_result()
366378

379+
def normalize_test_name(self, test_full_name, *, is_error=False):
380+
short_name = test_full_name.split(" ")[0]
381+
if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
382+
# This means that we have a failure in a life-cycle hook,
383+
# we need to rerun the whole module or class suite.
384+
# Basically the error looks like this:
385+
# ERROR: setUpClass (test.test_reg_ex.RegTest)
386+
# or
387+
# ERROR: setUpModule (test.test_reg_ex)
388+
# So, we need to parse the class / module name.
389+
lpar = test_full_name.index('(')
390+
rpar = test_full_name.index(')')
391+
return test_full_name[lpar + 1: rpar].split('.')[-1]
392+
return short_name
393+
367394
def display_result(self):
368395
# If running the test suite for PGO then no one cares about results.
369396
if self.ns.pgo:

‎Lib/test/support/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1108,7 +1108,7 @@ def _run_suite(suite):
11081108
if junit_xml_list is not None:
11091109
junit_xml_list.append(result.get_xml_element())
11101110

1111-
if not result.testsRun and not result.skipped:
1111+
if not result.testsRun and not result.skipped and not result.errors:
11121112
raise TestDidNotRun
11131113
if not result.wasSuccessful():
11141114
if len(result.errors) == 1 and not result.failures:

‎Lib/test/test_regrtest.py‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1120,6 +1120,160 @@ def test_fail_once(self):
11201120
self.check_executed_tests(output, [testname],
11211121
rerun={testname: "test_fail_once"})
11221122

1123+
def test_rerun_setup_class_hook_failure(self):
1124+
# FAILURE then FAILURE
1125+
code = textwrap.dedent("""
1126+
import unittest
1127+
1128+
class ExampleTests(unittest.TestCase):
1129+
@classmethod
1130+
def setUpClass(self):
1131+
raise RuntimeError('Fail')
1132+
1133+
def test_success(self):
1134+
return
1135+
""")
1136+
testname = self.create_test(code=code)
1137+
1138+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1139+
self.check_executed_tests(output, testname,
1140+
failed=[testname],
1141+
rerun={testname: "ExampleTests"})
1142+
1143+
def test_rerun_teardown_class_hook_failure(self):
1144+
# FAILURE then FAILURE
1145+
code = textwrap.dedent("""
1146+
import unittest
1147+
1148+
class ExampleTests(unittest.TestCase):
1149+
@classmethod
1150+
def tearDownClass(self):
1151+
raise RuntimeError('Fail')
1152+
1153+
def test_success(self):
1154+
return
1155+
""")
1156+
testname = self.create_test(code=code)
1157+
1158+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1159+
self.check_executed_tests(output, testname,
1160+
failed=[testname],
1161+
rerun={testname: "ExampleTests"})
1162+
1163+
def test_rerun_setup_module_hook_failure(self):
1164+
# FAILURE then FAILURE
1165+
code = textwrap.dedent("""
1166+
import unittest
1167+
1168+
def setUpModule():
1169+
raise RuntimeError('Fail')
1170+
1171+
class ExampleTests(unittest.TestCase):
1172+
def test_success(self):
1173+
return
1174+
""")
1175+
testname = self.create_test(code=code)
1176+
1177+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1178+
self.check_executed_tests(output, testname,
1179+
failed=[testname],
1180+
rerun={testname: testname})
1181+
1182+
def test_rerun_teardown_module_hook_failure(self):
1183+
# FAILURE then FAILURE
1184+
code = textwrap.dedent("""
1185+
import unittest
1186+
1187+
def tearDownModule():
1188+
raise RuntimeError('Fail')
1189+
1190+
class ExampleTests(unittest.TestCase):
1191+
def test_success(self):
1192+
return
1193+
""")
1194+
testname = self.create_test(code=code)
1195+
1196+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1197+
self.check_executed_tests(output, testname,
1198+
failed=[testname],
1199+
rerun={testname: testname})
1200+
1201+
def test_rerun_setup_hook_failure(self):
1202+
# FAILURE then FAILURE
1203+
code = textwrap.dedent("""
1204+
import unittest
1205+
1206+
class ExampleTests(unittest.TestCase):
1207+
def setUp(self):
1208+
raise RuntimeError('Fail')
1209+
1210+
def test_success(self):
1211+
return
1212+
""")
1213+
testname = self.create_test(code=code)
1214+
1215+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1216+
self.check_executed_tests(output, testname,
1217+
failed=[testname],
1218+
rerun={testname: "test_success"})
1219+
1220+
def test_rerun_teardown_hook_failure(self):
1221+
# FAILURE then FAILURE
1222+
code = textwrap.dedent("""
1223+
import unittest
1224+
1225+
class ExampleTests(unittest.TestCase):
1226+
def tearDown(self):
1227+
raise RuntimeError('Fail')
1228+
1229+
def test_success(self):
1230+
return
1231+
""")
1232+
testname = self.create_test(code=code)
1233+
1234+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1235+
self.check_executed_tests(output, testname,
1236+
failed=[testname],
1237+
rerun={testname: "test_success"})
1238+
1239+
def test_rerun_async_setup_hook_failure(self):
1240+
# FAILURE then FAILURE
1241+
code = textwrap.dedent("""
1242+
import unittest
1243+
1244+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1245+
async def asyncSetUp(self):
1246+
raise RuntimeError('Fail')
1247+
1248+
async def test_success(self):
1249+
return
1250+
""")
1251+
testname = self.create_test(code=code)
1252+
1253+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1254+
self.check_executed_tests(output, testname,
1255+
failed=[testname],
1256+
rerun={testname: "test_success"})
1257+
1258+
def test_rerun_async_teardown_hook_failure(self):
1259+
# FAILURE then FAILURE
1260+
code = textwrap.dedent("""
1261+
import unittest
1262+
1263+
class ExampleTests(unittest.IsolatedAsyncioTestCase):
1264+
async def asyncTearDown(self):
1265+
raise RuntimeError('Fail')
1266+
1267+
async def test_success(self):
1268+
return
1269+
""")
1270+
testname = self.create_test(code=code)
1271+
1272+
output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
1273+
self.check_executed_tests(output, testname,
1274+
failed=[testname],
1275+
rerun={testname: "test_success"})
1276+
11231277
def test_no_tests_ran(self):
11241278
code = textwrap.dedent("""
11251279
import unittest

0 commit comments

Comments
 (0)