gh-115421: Test that our Makefile has all needed test folders

Author: sobolevn

Lib/test/test_tools/test_makefile.py

@@ -0,0 +1,67 @@
+"""
+Tests for `Makefile`.
+"""
+
+import os
+import unittest
+from test import support
+
+MAKEFILE = os.path.join(support.REPO_ROOT, 'Makefile')
+
+if not support.check_impl_detail(cpython=True):
+    raise unittest.SkipTest('cpython only')
+if support.MS_WINDOWS:
+    raise unittest.SkipTest('Windows does not support make')
+if not os.path.exists(MAKEFILE) or not os.path.isfile(MAKEFILE):
+    raise unittest.SkipTest('Makefile could not be found')
+
+try:
+    import _testcapi
+except ImportError:
+    TEST_MODULES = False
+else:
+    TEST_MODULES = True
+
+@support.requires_subprocess()
+class TestMakefile(unittest.TestCase):
+    def list_test_dirs(self):
+        import subprocess
+        return subprocess.check_output(
+            ['make', '-C', support.REPO_ROOT, 'listtestmodules'],
+            universal_newlines=True,
+            encoding='utf-8',
+        )
+
+    def check_existing_test_modules(self, result):
+        test_dirs = result.split(' ')
+        unique_test_dirs = set(test_dirs)
+        self.assertEqual(len(test_dirs), len(unique_test_dirs))
+
+        idle_test = 'idlelib/idle_test'
+        self.assertIn(idle_test, test_dirs)
+
+        used = [idle_test]
+        for dirpath, _, _ in os.walk(support.TEST_HOME_DIR):
+            dirname = os.path.basename(dirpath)
+            if dirname == '__pycache__':
+                continue
+            with self.subTest(dirpath=dirpath):
+                relpath = os.path.relpath(dirpath, support.STDLIB_DIR)
+                self.assertIn(relpath, test_dirs)
+                used.append(relpath)
+
+        # Check that there are no extra entries:
+        self.assertEqual(
+            unique_test_dirs.symmetric_difference(set(used)),
+            set(),
+        )
+
+    def check_missing_test_modules(self, result):
+        self.assertEqual(result, '')
+
+    def test_makefile_test_folders(self):
+        result = self.list_test_dirs().strip()
+        if TEST_MODULES:
+            self.check_existing_test_modules(result)
+        else:
+            self.check_missing_test_modules(result)

Makefile.pre.in

@@ -2373,6 +2373,12 @@ COMPILEALL_OPTS=-j0
 
 TEST_MODULES=@TEST_MODULES@
 
+.PHONY: listtestmodules
+listtestmodules:
+	@if test "$(TEST_MODULES)" = yes; then \
+		echo "$(TESTSUBDIRS)"; \
+	fi
+
 .PHONY: libinstall
 libinstall:	all $(srcdir)/Modules/xxmodule.c
 	@for i in $(SCRIPTDIR) $(LIBDEST); \