bpo-45573: Move mandatory core modules to Modules/Setup.bootstrap (GH-29616)

Author: tiran

Makefile.pre.in

@@ -910,10 +910,12 @@ Makefile Modules/config.c: Makefile.pre \
 				$(MAKESETUP) \
 				$(srcdir)/Modules/Setup \
 				Modules/Setup.local \
+				$(srcdir)/Modules/Setup.bootstrap \
 				Modules/Setup.stdlib
 	$(SHELL) $(MAKESETUP) -c $(srcdir)/Modules/config.c.in \
 				-s Modules \
 				Modules/Setup.local \
+				$(srcdir)/Modules/Setup.bootstrap \
 				$(srcdir)/Modules/Setup
 	@mv config.c Modules
 	@echo "The Makefile was updated, you may need to re-run make."
@@ -2167,6 +2169,7 @@ libainstall:	@DEF_MAKE_RULE@ python-config
 	$(INSTALL_DATA) $(srcdir)/Modules/config.c.in $(DESTDIR)$(LIBPL)/config.c.in
 	$(INSTALL_DATA) Makefile $(DESTDIR)$(LIBPL)/Makefile
 	$(INSTALL_DATA) $(srcdir)/Modules/Setup $(DESTDIR)$(LIBPL)/Setup
+	$(INSTALL_DATA) $(srcdir)/Modules/Setup.bootstrap $(DESTDIR)$(LIBPL)/Setup.bootstrap
 	$(INSTALL_DATA) Modules/Setup.stdlib $(DESTDIR)$(LIBPL)/Setup.stdlib
 	$(INSTALL_DATA) Modules/Setup.local $(DESTDIR)$(LIBPL)/Setup.local
 	$(INSTALL_DATA) Misc/python.pc $(DESTDIR)$(LIBPC)/python-$(VERSION).pc

Misc/NEWS.d/next/Build/2021-11-18-13-31-02.bpo-45573.LCjGB8.rst

@@ -0,0 +1,2 @@
+Mandatory core modules, that are required to bootstrap Python, are now in
+``Modules/Setup.bootstrap``.

Modules/Setup

@@ -111,31 +111,8 @@ PYTHONPATH=$(COREPYTHONPATH)
 
 
 # ---
-# Built-in modules required to get a functioning interpreter;
-# cannot be built as shared!
-
-_collections _collectionsmodule.c
-_abc _abc.c
-_codecs _codecsmodule.c
-_functools _functoolsmodule.c
-_io -I$(srcdir)/Modules/_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
-_locale _localemodule.c
-_operator _operator.c
-_signal signalmodule.c
-_sre _sre.c
-_stat _stat.c
-_symtable symtablemodule.c
-_thread _threadmodule.c
-_tracemalloc _tracemalloc.c  # See bpo-35053 as to why this is built in.
-_weakref _weakref.c
-atexit atexitmodule.c
-errno errnomodule.c
-faulthandler faulthandler.c
-itertools itertoolsmodule.c
-posix posixmodule.c
-pwd pwdmodule.c
-time timemodule.c
-
+# Built-in modules required to get a functioning interpreter are listed in
+# Modules/Setup.bootstrap.
 
 # ---
 # The rest of the modules listed in this file are all commented out by

Modules/Setup.bootstrap

@@ -0,0 +1,34 @@
+# -*- makefile -*-
+
+# ---
+# Built-in modules required to get a functioning interpreter;
+# cannot be built as shared!
+
+# module C APIs are used in core
+atexit atexitmodule.c
+faulthandler faulthandler.c
+posix posixmodule.c
+_signal signalmodule.c
+_tracemalloc _tracemalloc.c
+
+# modules used by importlib, deepfreeze, freeze, runpy, and sysconfig
+_codecs _codecsmodule.c
+_collections _collectionsmodule.c
+errno errnomodule.c
+_io _io/_iomodule.c _io/iobase.c _io/fileio.c _io/bytesio.c _io/bufferedio.c _io/textio.c _io/stringio.c
+itertools itertoolsmodule.c
+_sre _sre.c
+_thread _threadmodule.c
+time timemodule.c
+_weakref _weakref.c
+
+# commonly used core modules
+_abc _abc.c
+_functools _functoolsmodule.c
+_locale _localemodule.c
+_operator _operator.c
+_stat _stat.c
+_symtable symtablemodule.c
+
+# for systems without $HOME env, used by site._getuserbase()
+pwd pwdmodule.c

Modules/Setup.stdlib.in

@@ -29,6 +29,9 @@
 ############################################################################
 # Modules that should always be present (POSIX and Windows):
 
+# needs libm and on some platforms librt
+@MODULE__DATETIME_TRUE@_datetime _datetimemodule.c
+
 # _decimal uses libmpdec
 # either static libmpdec.a from Modules/_decimal/libmpdec or libmpdec.so
 # with ./configure --with-system-libmpdec

configure

@@ -634,6 +634,12 @@ MODULE_PYEXPAT_FALSE
 MODULE_PYEXPAT_TRUE
 MODULE_OSSAUDIODEV_FALSE
 MODULE_OSSAUDIODEV_TRUE
+MODULE__DATETIME_FALSE
+MODULE__DATETIME_TRUE
+MODULE_TIME_FALSE
+MODULE_TIME_TRUE
+MODULE__IO_FALSE
+MODULE__IO_TRUE
 TEST_MODULES
 LIBRARY_DEPS
 STATIC_LIBPYTHON
@@ -19456,6 +19462,52 @@ MODULE_BLOCK=
 
 
 
+
+
+     if true; then
+  MODULE__IO_TRUE=
+  MODULE__IO_FALSE='#'
+else
+  MODULE__IO_TRUE='#'
+  MODULE__IO_FALSE=
+fi
+
+
+    as_fn_append MODULE_BLOCK "MODULE__IO_CFLAGS=-I\$(srcdir)/Modules/_io$as_nl"
+
+
+
+
+     if true; then
+  MODULE_TIME_TRUE=
+  MODULE_TIME_FALSE='#'
+else
+  MODULE_TIME_TRUE='#'
+  MODULE_TIME_FALSE=
+fi
+
+
+
+    as_fn_append MODULE_BLOCK "MODULE_TIME_LDFLAGS=$TIMEMODULE_LIB$as_nl"
+
+
+
+
+     if true; then
+  MODULE__DATETIME_TRUE=
+  MODULE__DATETIME_FALSE='#'
+else
+  MODULE__DATETIME_TRUE='#'
+  MODULE__DATETIME_FALSE=
+fi
+
+
+
+    as_fn_append MODULE_BLOCK "MODULE__DATETIME_LDFLAGS=$TIMEMODULE_LIB $LIBM$as_nl"
+
+
+
+
   { $as_echo "$as_me:${as_lineno-$LINENO}: checking for stdlib extension module ossaudiodev" >&5
 $as_echo_n "checking for stdlib extension module ossaudiodev... " >&6; }
       case $py_stdlib_not_available in #(
@@ -19828,6 +19880,18 @@ LTLIBOBJS=$ac_ltlibobjs
 
 
 
+if test -z "${MODULE__IO_TRUE}" && test -z "${MODULE__IO_FALSE}"; then
+  as_fn_error $? "conditional \"MODULE__IO\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${MODULE_TIME_TRUE}" && test -z "${MODULE_TIME_FALSE}"; then
+  as_fn_error $? "conditional \"MODULE_TIME\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
+if test -z "${MODULE__DATETIME_TRUE}" && test -z "${MODULE__DATETIME_FALSE}"; then
+  as_fn_error $? "conditional \"MODULE__DATETIME\" was never defined.
+Usually this means the macro was only invoked conditionally." "$LINENO" 5
+fi
 if test -z "${MODULE_OSSAUDIODEV_TRUE}" && test -z "${MODULE_OSSAUDIODEV_FALSE}"; then
   as_fn_error $? "conditional \"MODULE_OSSAUDIODEV\" was never defined.
 Usually this means the macro was only invoked conditionally." "$LINENO" 5
@@ -21068,7 +21132,7 @@ fi
 echo "creating Makefile" >&6
 $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
 			-s Modules \
-			Modules/Setup.local $srcdir/Modules/Setup
+			Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
 mv config.c Modules
 
 if test -z "$PKG_CONFIG"; then

configure.ac

@@ -6030,6 +6030,28 @@ AC_DEFUN([PY_STDLIB_MOD], [
   m4_popdef([modstate])dnl
 ])
 
+dnl Define simple, always enabled stdlib extension module
+dnl PY_STDLIB_MOD_SIMPLE([NAME], [CFLAGS], [LDFLAGS])
+dnl cflags and ldflags are optional
+AC_DEFUN([PY_STDLIB_MOD_SIMPLE], [
+  m4_pushdef([modcond], [MODULE_]m4_toupper([$1]))dnl
+  AM_CONDITIONAL(modcond, [true])
+  m4_ifval([$2], [
+    _MODULE_BLOCK_ADD([MODULE_]m4_toupper([$1])[_CFLAGS], [$2])
+  ])
+  m4_ifval([$3], [
+    _MODULE_BLOCK_ADD([MODULE_]m4_toupper([$1])[_LDFLAGS], [$3])
+  ])
+  m4_popdef([modcond])dnl
+])
+
+dnl static modules in Modules/Setup.bootstrap
+PY_STDLIB_MOD_SIMPLE([_io], [-I\$(srcdir)/Modules/_io], [])
+PY_STDLIB_MOD_SIMPLE([time], [], [$TIMEMODULE_LIB])
+
+dnl always enabled extension modules
+PY_STDLIB_MOD_SIMPLE([_datetime], [], [$TIMEMODULE_LIB $LIBM])
+
 dnl platform specific extensions
 PY_STDLIB_MOD([ossaudiodev],
   [], [test "$ac_cv_header_linux_soundcard_h" = yes -o "$ac_cv_header_sys_soundcard_h" = yes])
@@ -6062,7 +6084,7 @@ fi
 echo "creating Makefile" >&AS_MESSAGE_FD
 $SHELL $srcdir/Modules/makesetup -c $srcdir/Modules/config.c.in \
 			-s Modules \
-			Modules/Setup.local $srcdir/Modules/Setup
+			Modules/Setup.local $srcdir/Modules/Setup.bootstrap $srcdir/Modules/Setup
 mv config.c Modules
 
 if test -z "$PKG_CONFIG"; then

setup.py

@@ -1006,13 +1006,9 @@ def detect_simple_extensions(self):
         if lib:
             time_libs.append(lib)
 
-        # time operations and variables
-        self.add(Extension('time', ['timemodule.c'],
-                           libraries=time_libs))
         # libm is needed by delta_new() that uses round() and by accum() that
         # uses modf().
-        self.add(Extension('_datetime', ['_datetimemodule.c'],
-                           libraries=['m']))
+        self.addext(Extension('_datetime', ['_datetimemodule.c']))
         # zoneinfo module
         self.add(Extension('_zoneinfo', ['_zoneinfo.c']))
         # random number generator implemented in C
@@ -1034,8 +1030,6 @@ def detect_simple_extensions(self):
         self.add(Extension('_opcode', ['_opcode.c']))
         # asyncio speedups
         self.add(Extension("_asyncio", ["_asynciomodule.c"]))
-        # _abc speedups
-        self.add(Extension("_abc", ["_abc.c"]))
         # _queue module
         self.add(Extension("_queue", ["_queuemodule.c"]))
         # _statistics module
@@ -1054,8 +1048,6 @@ def detect_simple_extensions(self):
             libs = ['bsd']
         self.add(Extension('fcntl', ['fcntlmodule.c'],
                            libraries=libs))
-        # pwd(3)
-        self.add(Extension('pwd', ['pwdmodule.c']))
         # grp(3)
         if not VXWORKS:
             self.add(Extension('grp', ['grpmodule.c']))
@@ -1510,6 +1502,9 @@ def detect_modules(self):
         self.configure_compiler()
         self.init_inc_lib_dirs()
 
+        # Some C extensions are built by entries in Modules/Setup.bootstrap.
+        # These are extensions are required to bootstrap the interpreter or
+        # build process.
         self.detect_simple_extensions()
         if TEST_EXTENSIONS:
             self.detect_test_extensions()