Statically link libpython into interpreter (but keep building libpython3.x.so)

cpython-unix/build-cpython.sh

@@ -298,15 +298,12 @@ if [ -n "${PYTHON_MEETS_MAXIMUM_VERSION_3_10}" ]; then
     patch -p1 -i ${ROOT}/patch-configure-crypt-no-modify-libs.patch
 fi
 
-# We patched configure.ac above. Reflect those changes.
-autoconf
-
-# configure assumes cross compiling when host != target and doesn't provide a way to
-# override. Our target triple normalization may lead configure into thinking we
-# aren't cross-compiling when we are. So force a static "yes" value when our
-# build system says we are cross-compiling.
-if [ -n "${CROSS_COMPILING}" ]; then
-  patch -p1 -i ${ROOT}/patch-force-cross-compile.patch
+# Build a libpython3.x.so, but statically link the interpreter against
+# libpython.
+if [ -n "${PYTHON_MEETS_MINIMUM_VERSION_3_12}" ]; then
+    patch -p1 -i "${ROOT}/patch-python-configure-add-enable-static-libpython-for-interpreter.patch"
+else
+    patch -p1 -i "${ROOT}/patch-python-configure-add-enable-static-libpython-for-interpreter-${PYTHON_MAJMIN_VERSION}.patch"
 fi
 
 # BOLT instrumented binaries segfault in some test_embed tests for unknown reasons.
@@ -382,6 +379,7 @@ CONFIGURE_FLAGS="
     --with-system-expat
     --with-system-libmpdec
     --without-ensurepip
+    --enable-static-libpython-for-interpreter
     ${EXTRA_CONFIGURE_FLAGS}"
 
 
@@ -573,6 +571,14 @@ else
 fi
 
 if [ -n "${CROSS_COMPILING}" ]; then
+    # configure assumes cross compiling when host != target and doesn't
+    # provide a way to override. Our target triple normalization may
+    # lead configure into thinking we aren't cross-compiling when we
+    # are. So force a static "yes" value when our build system says we
+    # are cross-compiling.
+    # See also https://savannah.gnu.org/support/?110348
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} cross_compiling=yes"
+
     # configure doesn't like a handful of scenarios when cross-compiling.
     #
     # getaddrinfo buggy test fails for some reason. So we short-circuit it.
@@ -589,8 +595,18 @@ if [ -n "${CROSS_COMPILING}" ]; then
     if [ "${PYBUILD_PLATFORM}" != "macos" ]; then
         CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_working_tzset=yes"
     fi
+
+    # Also, it cannot detect whether the compiler supports -pthread or
+    # not, and conservatively defaults to no, which is not the right
+    # default on relatively modern compilers.
+    CONFIGURE_FLAGS="${CONFIGURE_FLAGS} ac_cv_pthread=yes"
+
+    # TODO: There are probably more of these, see #399.
 fi
 
+# We patched configure.ac above. Reflect those changes.
+autoconf
+
 CFLAGS=$CFLAGS CPPFLAGS=$CFLAGS LDFLAGS=$LDFLAGS \
     ./configure ${CONFIGURE_FLAGS}
 

cpython-unix/build.py

@@ -562,21 +562,34 @@ def python_build_info(
     bi["object_file_format"] = object_file_format
 
     # Determine allowed libaries on Linux
+    libs = extra_metadata["python_config_vars"].get("LIBS", "").split()
     mips = target_triple.split("-")[0] in {"mips", "mipsel"}
     linux_allowed_system_libraries = LINUX_ALLOW_SYSTEM_LIBRARIES.copy()
     if mips and version == "3.13":
         # See https://github.com/astral-sh/python-build-standalone/issues/410
         linux_allowed_system_libraries.add("atomic")
     riscv = target_triple.split("-")[0] in {"riscv64"}
     if riscv:
-        # RISC-V binary often comes with libatomic on old GCC versions
+        # On older GCC versions, RISC-V sub-word atomic operations require a
+        # helper function found in libatomic. To facilitate this, GCC <15 adds
+        # "-latomic" to the definition of "-pthread". We think it's generally
+        # reasonable on RISC-V systems (but not all Linux systems in general)
+        # to expect a libatomic system library is installed.
+        #
+        # Because "-latomic" is implicitly added by "-pthread", it may not be
+        # found in the LIBS sysconfig variable, but we need to pretend it is so
+        # that it gets into PYTHON.json (in particular, so that the validation
+        # script accepts this dependency).
+        #
         # See https://github.com/riscvarchive/riscv-gcc/issues/12
         # https://github.com/riscvarchive/riscv-gcc/issues/337
         # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86005
+        # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=104338
+        # https://github.com/gcc-mirror/gcc/commit/203f3060dd363361b172f7295f42bb6bf5ac0b3b
         linux_allowed_system_libraries.add("atomic")
+        libs.append("-latomic")
 
     # Add in core linking annotations.
-    libs = extra_metadata["python_config_vars"].get("LIBS", "").split()
     skip = False
     for i, lib in enumerate(libs):
         if skip:

cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter-3.10.patch

@@ -0,0 +1,93 @@
+From 579a7cf9498ccfa656dd720a5db8dd6e04e97150 Mon Sep 17 00:00:00 2001
+From: Geoffrey Thomas <geofft@ldpreload.com>
+Date: Sat, 19 Apr 2025 11:13:40 -0400
+Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
+
+This option changes the behavior of --enable-shared to continue to build
+the libpython3.x.so shared library, but not use it for linking the
+python3 interpreter executable. Instead, the executable is linked
+directly against the libpython .o files as it would be with
+--disable-shared [in newer versions of Python].
+
+There are two benefits of this change. First, libpython uses
+thread-local storage, which is noticeably slower when used in a loaded
+module instead of in the main program, because the main program can take
+advantage of constant offsets from the thread state pointer but loaded
+modules have to dynamically call a function __tls_get_addr() to
+potentially allocate their thread-local storage area. (There is another
+thread-local storage model for dynamic libraries which mitigates most of
+this performance hit, but it comes at the cost of preventing
+dlopen("libpython3.x.so"), which is a use case we want to preserve.)
+
+Second, this improves the user experience around relocatable Python a
+little bit, in that we don't need to use an $ORIGIN-relative path to
+locate libpython3.x.so, which has some mild benefits around musl (which
+does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
+DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
+setcap (which prevents processing $ORIGIN), etc.
+---
+ Makefile.pre.in |  4 +++-
+ configure.ac    | 18 ++++++++++++++++++
+ 2 files changed, 21 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index fa99dd86c41..84c00a5c071 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -458,6 +458,8 @@ LIBRARY_OBJS=	\
+ 		$(LIBRARY_OBJS_OMIT_FROZEN) \
+ 		Python/frozen.o
+
++LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
++
+ ##########################################################################
+ # DTrace
+
+@@ -586,7 +588,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
+
+ # Build the interpreter
+ $(BUILDPYTHON):	Programs/python.o $(LIBRARY_DEPS)
+-	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
++	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
+
+ platform: $(BUILDPYTHON) pybuilddir.txt
+ 	$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
+diff --git a/configure.ac b/configure.ac
+index ac3be3850a9..a07003a24ed 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1093,6 +1093,17 @@ then
+ fi
+ AC_MSG_RESULT($enable_shared)
+
++AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
++AC_ARG_ENABLE([static-libpython-for-interpreter],
++   AS_HELP_STRING([--enable-static-libpython-for-interpreter],
++                  [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
++
++if test -z "$enable_static_libpython_for_interpreter"
++then
++  enable_static_libpython_for_interpreter="no"
++fi
++AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
++
+ AC_MSG_CHECKING(for --enable-profiling)
+ AC_ARG_ENABLE(profiling,
+               AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
+@@ -1198,6 +1209,13 @@ fi
+
+ AC_MSG_RESULT($LDLIBRARY)
+
++if test "$enable_static_libpython_for_interpreter" = "yes"; then
++    LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
++else
++    LINK_PYTHON_OBJS='$(BLDLIBRARY)'
++fi
++AC_SUBST(LINK_PYTHON_OBJS)
++
+ AC_SUBST(AR)
+ AC_CHECK_TOOLS(AR, ar aal, ar)
+
+-- 
+2.39.5 (Apple Git-154)
+

cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter-3.11.patch

@@ -0,0 +1,69 @@
+From a5182aec2c0597adb8a01298af120809fcf3187b Mon Sep 17 00:00:00 2001
+From: Geoffrey Thomas <geofft@ldpreload.com>
+Date: Sat, 19 Apr 2025 11:13:40 -0400
+Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
+
+This option changes the behavior of --enable-shared to continue to build
+the libpython3.x.so shared library, but not use it for linking the
+python3 interpreter executable. Instead, the executable is linked
+directly against the libpython .o files as it would be with
+--disable-shared.
+
+There are two benefits of this change. First, libpython uses
+thread-local storage, which is noticeably slower when used in a loaded
+module instead of in the main program, because the main program can take
+advantage of constant offsets from the thread state pointer but loaded
+modules have to dynamically call a function __tls_get_addr() to
+potentially allocate their thread-local storage area. (There is another
+thread-local storage model for dynamic libraries which mitigates most of
+this performance hit, but it comes at the cost of preventing
+dlopen("libpython3.x.so"), which is a use case we want to preserve.)
+
+Second, this improves the user experience around relocatable Python a
+little bit, in that we don't need to use an $ORIGIN-relative path to
+locate libpython3.x.so, which has some mild benefits around musl (which
+does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
+DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
+setcap (which prevents processing $ORIGIN), etc.
+---
+ configure.ac | 17 ++++++++++++++++-
+ 1 file changed, 16 insertions(+), 1 deletion(-)
+
+diff --git a/configure.ac b/configure.ac
+index ab5e1de6fab..6783c36da4d 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1419,6 +1419,17 @@ fi],
+ [AC_MSG_RESULT(yes)])
+ AC_SUBST(STATIC_LIBPYTHON)
+
++AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
++AC_ARG_ENABLE([static-libpython-for-interpreter],
++   AS_HELP_STRING([--enable-static-libpython-for-interpreter],
++                  [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
++
++if test -z "$enable_static_libpython_for_interpreter"
++then
++  enable_static_libpython_for_interpreter="no"
++fi
++AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
++
+ AC_MSG_CHECKING(for --enable-profiling)
+ AC_ARG_ENABLE(profiling,
+               AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
+@@ -1563,7 +1574,11 @@ if test "$PY_ENABLE_SHARED" = 1 || test "$enable_framework" ; then
+         LIBRARY_DEPS="\$(LIBRARY) $LIBRARY_DEPS"
+     fi
+     # Link Python program to the shared library
+-    LINK_PYTHON_OBJS='$(BLDLIBRARY)'
++    if test "$enable_static_libpython_for_interpreter" = "yes"; then
++        LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
++    else
++        LINK_PYTHON_OBJS='$(BLDLIBRARY)'
++    fi
+ else
+     if test "$STATIC_LIBPYTHON" = 0; then
+         # Build Python needs object files but don't need to build
+-- 
+2.39.5 (Apple Git-154)
+

cpython-unix/patch-python-configure-add-enable-static-libpython-for-interpreter-3.9.patch

@@ -0,0 +1,93 @@
+From 5ae9112a87d45c3aff5ee269ff8e2e49ca278ed3 Mon Sep 17 00:00:00 2001
+From: Geoffrey Thomas <geofft@ldpreload.com>
+Date: Sat, 19 Apr 2025 11:13:40 -0400
+Subject: [PATCH 1/1] configure: add --enable-static-libpython-for-interpreter
+
+This option changes the behavior of --enable-shared to continue to build
+the libpython3.x.so shared library, but not use it for linking the
+python3 interpreter executable. Instead, the executable is linked
+directly against the libpython .o files as it would be with
+--disable-shared [in newer versions of Python].
+
+There are two benefits of this change. First, libpython uses
+thread-local storage, which is noticeably slower when used in a loaded
+module instead of in the main program, because the main program can take
+advantage of constant offsets from the thread state pointer but loaded
+modules have to dynamically call a function __tls_get_addr() to
+potentially allocate their thread-local storage area. (There is another
+thread-local storage model for dynamic libraries which mitigates most of
+this performance hit, but it comes at the cost of preventing
+dlopen("libpython3.x.so"), which is a use case we want to preserve.)
+
+Second, this improves the user experience around relocatable Python a
+little bit, in that we don't need to use an $ORIGIN-relative path to
+locate libpython3.x.so, which has some mild benefits around musl (which
+does not support $ORIGIN-relative DT_NEEDED, only $ORIGIN-relative
+DT_RPATH/DT_RUNPATH), users who want to make the interpreter setuid or
+setcap (which prevents processing $ORIGIN), etc.
+---
+ Makefile.pre.in |  4 +++-
+ configure.ac    | 18 ++++++++++++++++++
+ 2 files changed, 21 insertions(+), 1 deletion(-)
+
+diff --git a/Makefile.pre.in b/Makefile.pre.in
+index a276d535c7f..193439aa73e 100644
+--- a/Makefile.pre.in
++++ b/Makefile.pre.in
+@@ -460,6 +460,8 @@ LIBRARY_OBJS=	\
+ 		$(LIBRARY_OBJS_OMIT_FROZEN) \
+ 		Python/frozen.o
+
++LINK_PYTHON_OBJS=@LINK_PYTHON_OBJS@
++
+ ##########################################################################
+ # DTrace
+
+@@ -589,7 +591,7 @@ clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
+
+ # Build the interpreter
+ $(BUILDPYTHON):	Programs/python.o $(LIBRARY) $(LDLIBRARY) $(PY3LIBRARY) $(EXPORTSYMS)
+-	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
++	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/python.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
+
+ platform: $(BUILDPYTHON) pybuilddir.txt
+ 	$(RUNSHARED) $(PYTHON_FOR_BUILD) -c 'import sys ; from sysconfig import get_platform ; print("%s-%d.%d" % (get_platform(), *sys.version_info[:2]))' >platform
+diff --git a/configure.ac b/configure.ac
+index aa515da4655..122b11def62 100644
+--- a/configure.ac
++++ b/configure.ac
+@@ -1106,6 +1106,17 @@ then
+ fi
+ AC_MSG_RESULT($enable_shared)
+
++AC_MSG_CHECKING([for --enable-static-libpython-for-interpreter])
++AC_ARG_ENABLE([static-libpython-for-interpreter],
++   AS_HELP_STRING([--enable-static-libpython-for-interpreter],
++                  [even with --enable-shared, statically link libpython into the interpreter (default is to use the shared library)]))
++
++if test -z "$enable_static_libpython_for_interpreter"
++then
++  enable_static_libpython_for_interpreter="no"
++fi
++AC_MSG_RESULT([$enable_static_libpython_for_interpreter])
++
+ AC_MSG_CHECKING(for --enable-profiling)
+ AC_ARG_ENABLE(profiling,
+               AS_HELP_STRING([--enable-profiling], [enable C-level code profiling with gprof (default is no)]))
+@@ -1211,6 +1222,13 @@ fi
+
+ AC_MSG_RESULT($LDLIBRARY)
+
++if test "$enable_static_libpython_for_interpreter" = "yes"; then
++    LINK_PYTHON_OBJS='$(LIBRARY_OBJS)'
++else
++    LINK_PYTHON_OBJS='$(BLDLIBRARY)'
++fi
++AC_SUBST(LINK_PYTHON_OBJS)
++
+ AC_SUBST(AR)
+ AC_CHECK_TOOLS(AR, ar aal, ar)
+
+-- 
+2.39.5 (Apple Git-154)
+