[Python-checkins] [3.11] bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (GH-30895) (GH-103342)
ambv
webhook-mailer at python.org
Fri Apr 7 13:35:24 EDT 2023
https://github.com/python/cpython/commit/ecb09a849689764193e0115d27e220f82b5f6d9f
commit: ecb09a849689764193e0115d27e220f82b5f6d9f
branch: 3.11
author: Łukasz Langa <lukasz at langa.pl>
committer: ambv <lukasz at langa.pl>
date: 2023-04-07T19:35:16+02:00
summary:
[3.11] bpo-46523: fix tests rerun when `setUp[Class|Module]` fails (GH-30895) (GH-103342)
(cherry picked from commit 995386071f96e4cfebfa027a71ca9134e4651d2a)
Co-authored-by: Nikita Sobolev <mail at sobolevn.me>
files:
M Lib/test/libregrtest/main.py
M Lib/test/support/__init__.py
M Lib/test/test_regrtest.py
diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 655e4d2e56f8..0125227bf117 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -28,6 +28,19 @@
# Must be smaller than buildbot "1200 seconds without output" limit.
EXIT_TIMEOUT = 120.0
+# gh-90681: When rerunning tests, we might need to rerun the whole
+# class or module suite if some its life-cycle hooks fail.
+# Test level hooks are not affected.
+_TEST_LIFECYCLE_HOOKS = frozenset((
+ 'setUpClass', 'tearDownClass',
+ 'setUpModule', 'tearDownModule',
+))
+
+EXITCODE_BAD_TEST = 2
+EXITCODE_INTERRUPTED = 130
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+
class Regrtest:
"""Execute a test suite.
@@ -331,8 +344,12 @@ def rerun_failed_tests(self):
errors = result.errors or []
failures = result.failures or []
- error_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in errors]
- failure_names = [test_full_name.split(" ")[0] for (test_full_name, *_) in failures]
+ error_names = [
+ self.normalize_test_name(test_full_name, is_error=True)
+ for (test_full_name, *_) in errors]
+ failure_names = [
+ self.normalize_test_name(test_full_name)
+ for (test_full_name, *_) in failures]
self.ns.verbose = True
orig_match_tests = self.ns.match_tests
if errors or failures:
@@ -358,6 +375,21 @@ def rerun_failed_tests(self):
self.display_result()
+ def normalize_test_name(self, test_full_name, *, is_error=False):
+ short_name = test_full_name.split(" ")[0]
+ if is_error and short_name in _TEST_LIFECYCLE_HOOKS:
+ # This means that we have a failure in a life-cycle hook,
+ # we need to rerun the whole module or class suite.
+ # Basically the error looks like this:
+ # ERROR: setUpClass (test.test_reg_ex.RegTest)
+ # or
+ # ERROR: setUpModule (test.test_reg_ex)
+ # So, we need to parse the class / module name.
+ lpar = test_full_name.index('(')
+ rpar = test_full_name.index(')')
+ return test_full_name[lpar + 1: rpar].split('.')[-1]
+ return short_name
+
def display_result(self):
# If running the test suite for PGO then no one cares about results.
if self.ns.pgo:
diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py
index c33f90d8071d..98be9cdd0e16 100644
--- a/Lib/test/support/__init__.py
+++ b/Lib/test/support/__init__.py
@@ -1097,7 +1097,7 @@ def _run_suite(suite):
if junit_xml_list is not None:
junit_xml_list.append(result.get_xml_element())
- if not result.testsRun and not result.skipped:
+ if not result.testsRun and not result.skipped and not result.errors:
raise TestDidNotRun
if not result.wasSuccessful():
if len(result.errors) == 1 and not result.failures:
diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 15e2f89ee20c..f692dd1e3e65 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -30,6 +30,11 @@
ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
+EXITCODE_BAD_TEST = 2
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+EXITCODE_INTERRUPTED = 130
+
TEST_INTERRUPTED = textwrap.dedent("""
from signal import SIGINT, raise_signal
try:
@@ -1115,6 +1120,160 @@ def test_fail_once(self):
self.check_executed_tests(output, [testname],
rerun={testname: "test_fail_once"})
+ def test_rerun_setup_class_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ class ExampleTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(self):
+ raise RuntimeError('Fail')
+
+ def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: "ExampleTests"})
+
+ def test_rerun_teardown_class_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ class ExampleTests(unittest.TestCase):
+ @classmethod
+ def tearDownClass(self):
+ raise RuntimeError('Fail')
+
+ def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: "ExampleTests"})
+
+ def test_rerun_setup_module_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ def setUpModule():
+ raise RuntimeError('Fail')
+
+ class ExampleTests(unittest.TestCase):
+ def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: testname})
+
+ def test_rerun_teardown_module_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ def tearDownModule():
+ raise RuntimeError('Fail')
+
+ class ExampleTests(unittest.TestCase):
+ def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: testname})
+
+ def test_rerun_setup_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ class ExampleTests(unittest.TestCase):
+ def setUp(self):
+ raise RuntimeError('Fail')
+
+ def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: "test_success"})
+
+ def test_rerun_teardown_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ class ExampleTests(unittest.TestCase):
+ def tearDown(self):
+ raise RuntimeError('Fail')
+
+ def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: "test_success"})
+
+ def test_rerun_async_setup_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ class ExampleTests(unittest.IsolatedAsyncioTestCase):
+ async def asyncSetUp(self):
+ raise RuntimeError('Fail')
+
+ async def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: "test_success"})
+
+ def test_rerun_async_teardown_hook_failure(self):
+ # FAILURE then FAILURE
+ code = textwrap.dedent("""
+ import unittest
+
+ class ExampleTests(unittest.IsolatedAsyncioTestCase):
+ async def asyncTearDown(self):
+ raise RuntimeError('Fail')
+
+ async def test_success(self):
+ return
+ """)
+ testname = self.create_test(code=code)
+
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
+ self.check_executed_tests(output, testname,
+ failed=[testname],
+ rerun={testname: "test_success"})
+
def test_no_tests_ran(self):
code = textwrap.dedent("""
import unittest
More information about the Python-checkins
mailing list