gh-93939: Build C extensions without setup.py (GH-94474)

Combines GH-93940, GH-94452, and GH-94433

Doc/whatsnew/3.12.rst

@@ -356,6 +356,12 @@ Changes in the Python API
 Build Changes
 =============
 
+* Python no longer uses ``setup.py`` to build shared C extension modules.
+  Build parameters like headers and libraries are detected in ``configure``
+  script. Extensions are built by ``Makefile``. Most extensions use
+  ``pkg-config`` and fall back to manual detection.
+  (Contributed by Christian Heimes in :gh:`93939`.)
+
 * ``va_start()`` with two parameters, like ``va_start(args, format),``
   is now required to build Python.
   ``va_start()`` is no longer called with a single parameter.

Lib/test/test_descr.py

@@ -21,6 +21,11 @@
 except ImportError:
     _testcapi = None
 
+try:
+    import xxsubtype
+except ImportError:
+    xxsubtype = None
+
 
 class OperatorsTest(unittest.TestCase):
 
@@ -299,6 +304,7 @@ def test_explicit_reverse_methods(self):
         self.assertEqual(float.__rsub__(3.0, 1), -2.0)
 
     @support.impl_detail("the module 'xxsubtype' is internal")
+    @unittest.skipIf(xxsubtype is None, "requires xxsubtype module")
     def test_spam_lists(self):
         # Testing spamlist operations...
         import copy, xxsubtype as spam
@@ -343,6 +349,7 @@ def foo(self): return 1
         self.assertEqual(a.getstate(), 42)
 
     @support.impl_detail("the module 'xxsubtype' is internal")
+    @unittest.skipIf(xxsubtype is None, "requires xxsubtype module")
     def test_spam_dicts(self):
         # Testing spamdict operations...
         import copy, xxsubtype as spam
@@ -1600,6 +1607,7 @@ def test_refleaks_in_classmethod___init__(self):
         self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10)
 
     @support.impl_detail("the module 'xxsubtype' is internal")
+    @unittest.skipIf(xxsubtype is None, "requires xxsubtype module")
     def test_classmethods_in_c(self):
         # Testing C-based class methods...
         import xxsubtype as spam
@@ -1683,6 +1691,7 @@ def test_refleaks_in_staticmethod___init__(self):
         self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10)
 
     @support.impl_detail("the module 'xxsubtype' is internal")
+    @unittest.skipIf(xxsubtype is None, "requires xxsubtype module")
     def test_staticmethods_in_c(self):
         # Testing C-based static methods...
         import xxsubtype as spam

Makefile.pre.in

@@ -211,12 +211,6 @@ ENSUREPIP=      @ENSUREPIP@
 LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
 LIBEXPAT_A= Modules/expat/libexpat.a
 
-# OpenSSL options for setup.py so sysconfig can pick up AC_SUBST() vars.
-OPENSSL_INCLUDES=@OPENSSL_INCLUDES@
-OPENSSL_LIBS=@OPENSSL_LIBS@
-OPENSSL_LDFLAGS=@OPENSSL_LDFLAGS@
-OPENSSL_RPATH=@OPENSSL_RPATH@
-
 # Module state, compiler flags and linker flags
 # Empty CFLAGS and LDFLAGS are omitted.
 # states:
@@ -582,9 +576,10 @@ LIBEXPAT_HEADERS= \
 
 # Default target
 all:		@DEF_MAKE_ALL_RULE@
-build_all:	check-clean-src $(BUILDPYTHON) platform oldsharedmods sharedmods \
-		gdbhooks Programs/_testembed scripts
-build_wasm: check-clean-src $(BUILDPYTHON) platform oldsharedmods python-config
+build_all:	check-clean-src $(BUILDPYTHON) platform sharedmods \
+		gdbhooks Programs/_testembed scripts checksharedmods
+build_wasm: check-clean-src $(BUILDPYTHON) platform sharedmods \
+		python-config checksharedmods
 
 # Check that the source is clean when building out of source.
 check-clean-src:
@@ -726,22 +721,6 @@ $(srcdir)/Modules/_blake2/blake2s_impl.c: $(srcdir)/Modules/_blake2/blake2b_impl
 	$(PYTHON_FOR_REGEN) $(srcdir)/Modules/_blake2/blake2b2s.py
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py -f $@
 
-# Build the shared modules
-# Under GNU make, MAKEFLAGS are sorted and normalized; the 's' for
-# -s, --silent or --quiet is always the first char.
-# Under BSD make, MAKEFLAGS might be " -s -v x=y".
-# Ignore macros passed by GNU make, passed after --
-sharedmods: $(PYTHON_FOR_BUILD_DEPS) pybuilddir.txt @LIBMPDEC_INTERNAL@ @LIBEXPAT_INTERNAL@
-	@case "`echo X $$MAKEFLAGS | sed 's/^X //;s/ -- .*//'`" in \
-	    *\ -s*|s*) quiet="-q";; \
-	    *) quiet="";; \
-	esac; \
-	echo "$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
-		$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build"; \
-	$(RUNSHARED) CC='$(CC)' LDSHARED='$(BLDSHARED)' OPT='$(OPT)' \
-		$(PYTHON_FOR_BUILD) $(srcdir)/setup.py $$quiet build
-
-
 # Build static library
 $(LIBRARY): $(LIBRARY_OBJS)
 	-rm -f $@
@@ -832,10 +811,6 @@ python.worker.js: $(srcdir)/Tools/wasm/python.worker.js
 # Build static libmpdec.a
 LIBMPDEC_CFLAGS=@LIBMPDEC_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED)
 
-# for setup.py
-DECIMAL_CFLAGS=@LIBMPDEC_CFLAGS@
-DECIMAL_LDFLAGS=@LIBMPDEC_LDFLAGS@
-
 # "%.o: %c" is not portable
 Modules/_decimal/libmpdec/basearith.o: $(srcdir)/Modules/_decimal/libmpdec/basearith.c $(LIBMPDEC_HEADERS) $(PYTHON_HEADERS)
 	$(CC) -c $(LIBMPDEC_CFLAGS) -o $@ $(srcdir)/Modules/_decimal/libmpdec/basearith.c
@@ -890,10 +865,6 @@ $(LIBMPDEC_A): $(LIBMPDEC_OBJS)
 # Build static libexpat.a
 LIBEXPAT_CFLAGS=@LIBEXPAT_CFLAGS@ $(PY_STDMODULE_CFLAGS) $(CCSHARED)
 
-# for setup.py
-EXPAT_CFLAGS=@LIBEXPAT_CFLAGS@
-EXPAT_LDFLAGS=@LIBEXPAT_LDFLAGS@
-
 Modules/expat/xmlparse.o: $(srcdir)/Modules/expat/xmlparse.c $(LIBEXPAT_HEADERS) $(PYTHON_HEADERS)
 	$(CC) -c $(LIBEXPAT_CFLAGS) -o $@ $(srcdir)/Modules/expat/xmlparse.c
 
@@ -910,7 +881,7 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
 # create relative links from build/lib.platform/egg.so to Modules/egg.so
 # pybuilddir.txt is created too late. We cannot use it in Makefile
 # targets. ln --relative is not portable.
-oldsharedmods: $(SHAREDMODS) pybuilddir.txt
+sharedmods: $(SHAREDMODS) pybuilddir.txt
 	@target=`cat pybuilddir.txt`; \
 	$(MKDIR_P) $$target; \
 	for mod in X $(SHAREDMODS); do \
@@ -919,7 +890,8 @@ oldsharedmods: $(SHAREDMODS) pybuilddir.txt
 		fi; \
 	done
 
-checksharedmods: oldsharedmods sharedmods $(PYTHON_FOR_BUILD_DEPS)
+# dependency on BUILDPYTHON ensures that the target is run last
+checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON)
 	@$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/scripts/check_extension_modules.py
 
 Modules/Setup.local:
@@ -942,7 +914,7 @@ Makefile Modules/config.c: Makefile.pre \
 	$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
 				-s Modules \
 				Modules/Setup.local \
-				@MODULES_SETUP_STDLIB@ \
+				Modules/Setup.stdlib \
 				Modules/Setup.bootstrap \
 				$(srcdir)/Modules/Setup
 	@mv config.c Modules
@@ -1762,13 +1734,13 @@ altinstall: commoninstall
 
 commoninstall:  check-clean-src @FRAMEWORKALTINSTALLFIRST@ \
 		altbininstall libinstall inclinstall libainstall \
-		sharedinstall oldsharedinstall altmaninstall \
+		sharedinstall altmaninstall \
 		@FRAMEWORKALTINSTALLLAST@
 
 # Install shared libraries enabled by Setup
 DESTDIRS=	$(exec_prefix) $(LIBDIR) $(BINLIBDEST) $(DESTSHARED)
 
-oldsharedinstall: $(DESTSHARED) all
+sharedinstall: $(DESTSHARED) all
 		@for i in X $(SHAREDMODS); do \
 		  if test $$i != X; then \
 		    echo $(INSTALL_SHARED) $$i $(DESTSHARED)/`basename $$i`; \
@@ -2252,17 +2224,6 @@ libainstall: all scripts
 	else true; \
 	fi
 
-# Install the dynamically loadable modules
-# This goes into $(exec_prefix)
-sharedinstall: all
-	$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/setup.py install \
-	   	--prefix=$(prefix) \
-		--install-scripts=$(BINDIR) \
-		--install-platlib=$(DESTSHARED) \
-		--root=$(DESTDIR)/
-	-rm $(DESTDIR)$(DESTSHARED)/_sysconfigdata_$(ABIFLAGS)_$(MACHDEP)_$(MULTIARCH).py
-	-rm -r $(DESTDIR)$(DESTSHARED)/__pycache__
-
 # Here are a couple of targets for MacOSX again, to install a full
 # framework-based Python. frameworkinstall installs everything, the
 # subtargets install specific parts. Much of the actual work is offloaded to
@@ -2536,10 +2497,10 @@ update-config:
 Python/thread.o: @THREADHEADERS@ $(srcdir)/Python/condvar.h
 
 # Declare targets that aren't real files
-.PHONY: all build_all build_wasm sharedmods check-clean-src
-.PHONY: oldsharedmods checksharedmods test quicktest
-.PHONY: install altinstall oldsharedinstall bininstall altbininstall
-.PHONY: maninstall libinstall inclinstall libainstall sharedinstall
+.PHONY: all build_all build_wasm check-clean-src
+.PHONY: sharedmods checksharedmods test quicktest
+.PHONY: install altinstall sharedinstall bininstall altbininstall
+.PHONY: maninstall libinstall inclinstall libainstall
 .PHONY: frameworkinstall frameworkinstallframework frameworkinstallstructure
 .PHONY: frameworkinstallmaclib frameworkinstallapps frameworkinstallunixtools
 .PHONY: frameworkaltinstallunixtools recheck clean clobber distclean

Misc/NEWS.d/next/Build/2022-06-28-09-42-10.gh-issue-93939._VWxKW.rst

@@ -0,0 +1,2 @@
+C extension modules are now built by ``configure`` and ``make``
+instead of :mod:`distutils` and ``setup.py``.

Modules/Setup

@@ -275,7 +275,7 @@ PYTHONPATH=$(COREPYTHONPATH)
 #xx xxmodule.c
 #xxlimited xxlimited.c
 #xxlimited_35 xxlimited_35.c
-xxsubtype xxsubtype.c  # Required for the test suite to pass!
+#xxsubtype xxsubtype.c
 
 # Testing
 

Modules/Setup.stdlib.in

@@ -165,6 +165,7 @@
 ############################################################################
 # Test modules
 
+@MODULE_XXSUBTYPE_TRUE@xxsubtype xxsubtype.c
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c

Tools/scripts/check_extension_modules.py

@@ -130,10 +130,11 @@ class ModuleChecker:
     pybuilddir_txt = "pybuilddir.txt"
 
     setup_files = (
-        SRC_DIR / "Modules/Setup",
+        # see end of configure.ac
         "Modules/Setup.local",
-        "Modules/Setup.bootstrap",
         "Modules/Setup.stdlib",
+        "Modules/Setup.bootstrap",
+        SRC_DIR / "Modules/Setup",
     )
 
     def __init__(self, cross_compiling: bool = False, strict: bool = False):
@@ -308,12 +309,6 @@ def get_sysconfig_modules(self) -> Iterable[ModuleInfo]:
         MODBUILT_NAMES: modules in *static* block
         MODSHARED_NAMES: modules in *shared* block
         MODDISABLED_NAMES: modules in *disabled* block
-
-        Modules built by setup.py addext() have a MODULE_{modname}_STATE entry,
-        but are not listed in MODSHARED_NAMES.
-
-        Modules built by old-style setup.py add() have neither a  MODULE_{modname}
-        entry nor an entry in MODSHARED_NAMES.
         """
         moddisabled = set(sysconfig.get_config_var("MODDISABLED_NAMES").split())
         if self.cross_compiling:

Tools/scripts/generate_stdlib_module_names.py

@@ -12,7 +12,6 @@
 
 SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 STDLIB_PATH = os.path.join(SRC_DIR, 'Lib')
-SETUP_PY = os.path.join(SRC_DIR, 'setup.py')
 
 IGNORE = {
     '__init__',
@@ -64,15 +63,6 @@ def list_packages(names):
             names.add(name)
 
 
-# Extension modules built by setup.py
-def list_setup_extensions(names):
-    cmd = [sys.executable, SETUP_PY, "-q", "build", "--list-module-names"]
-    output = subprocess.check_output(cmd)
-    output = output.decode("utf8")
-    extensions = output.splitlines()
-    names |= set(extensions)
-
-
 # Built-in and extension modules built by Modules/Setup*
 # includes Windows and macOS extensions.
 def list_modules_setup_extensions(names):
@@ -103,7 +93,6 @@ def list_frozen(names):
 def list_modules():
     names = set(sys.builtin_module_names)
     list_modules_setup_extensions(names)
-    list_setup_extensions(names)
     list_packages(names)
     list_python_modules(names)
     list_frozen(names)