[Python-checkins] bpo-37667: Add regression test for regrtest. (GH-14929)

[Gregory P. Smith]

https://github.com/python/cpython/commit/e95ac20103437d8099fb91cd4468ab030852f3b7
commit: e95ac20103437d8099fb91cd4468ab030852f3b7
branch: master
author: Gregory P. Smith <greg at krypto.org>
committer: GitHub <noreply at github.com>
date: 2019-07-23T22:28:48-07:00
summary:

bpo-37667: Add regression test for regrtest. (GH-14929)

Verify that it appears to find roughly the right number of tests in the stdlib's testsuite.

files:
M Lib/test/test_regrtest.py

diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py
index 2e5df00dc51b..fdab8c8ef5d1 100644
--- a/Lib/test/test_regrtest.py
+++ b/Lib/test/test_regrtest.py
@@ -6,6 +6,7 @@
 
 import contextlib
 import faulthandler
+import glob
 import io
 import os.path
 import platform
@@ -529,6 +530,31 @@ def run_python(self, args, **kw):
         return proc.stdout
 
 
+class CheckActualTests(BaseTestCase):
+    """
+    Check that regrtest appears to find the expected set of tests.
+    """
+
+    def test_finds_expected_number_of_tests(self):
+        args = ['-Wd', '-E', '-bb', '-m', 'test.regrtest', '--list-tests']
+        output = self.run_python(args)
+        rough_number_of_tests_found = len(output.splitlines())
+        actual_testsuite_glob = os.path.join(os.path.dirname(__file__),
+                                             'test*.py')
+        rough_counted_test_py_files = len(glob.glob(actual_testsuite_glob))
+        # We're not trying to duplicate test finding logic in here,
+        # just give a rough estimate of how many there should be and
+        # be near that.  This is a regression test to prevent mishaps
+        # such as https://bugs.python.org/issue37667 in the future.
+        # If you need to change the values in here during some
+        # mythical future test suite reorganization, don't go
+        # overboard with logic and keep that goal in mind.
+        self.assertGreater(rough_number_of_tests_found,
+                           rough_counted_test_py_files*9//10,
+                           msg='Unexpectedly low number of tests found in:\n'
+                           f'{", ".join(output.splitlines())}')
+
+
 class ProgramsTestCase(BaseTestCase):
     """
     Test various ways to run the Python test suite. Use options close