bpo-45696: Deep-freeze selected modules (pythonGH-29118)

This gains 10% or more in startup time for `python -c pass` on UNIX-ish systems.

The Makefile.pre.in generating code builds on Eric's work for bpo-45020, but the .c file generator is new.

Windows version TBD.

.gitattributes

@@ -46,6 +46,7 @@ Modules/clinic/*.h          linguist-generated=true
 Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
+Python/deepfreeze/*.c       linguist-generated=true
 Python/frozen_modules/*.h   linguist-generated=true
 Python/frozen_modules/MANIFEST  linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true

.gitignore

@@ -59,6 +59,7 @@ Lib/distutils/command/*.pdb
 Lib/lib2to3/*.pickle
 Lib/test/data/*
 !Lib/test/data/README
+/_bootstrap_python
 /Makefile
 /Makefile.pre
 Mac/Makefile
@@ -115,6 +116,7 @@ Tools/unicode/data/
 /platform
 /profile-clean-stamp
 /profile-run-stamp
+/Python/deepfreeze/*.c
 /pybuilddir.txt
 /pyconfig.h
 /python-config

Doc/library/ctypes.rst

@@ -1087,7 +1087,9 @@ size, we show only how this table can be read with :mod:`ctypes`::
    >>> class struct_frozen(Structure):
    ...     _fields_ = [("name", c_char_p),
    ...                 ("code", POINTER(c_ubyte)),
-   ...                 ("size", c_int)]
+   ...                 ("size", c_int),
+   ...                 ("get_code", POINTER(c_ubyte)),  # Function pointer
+   ...                ]
    ...
    >>>
 

Include/cpython/import.h

@@ -32,6 +32,7 @@ struct _frozen {
     const char *name;                 /* ASCII encoded string */
     const unsigned char *code;
     int size;
+    PyObject *(*get_code)(void);
 };
 
 /* Embedding apps may change this pointer to point to their favorite

Lib/ctypes/test/test_values.py

@@ -53,7 +53,9 @@ def test_frozentable(self):
         class struct_frozen(Structure):
             _fields_ = [("name", c_char_p),
                         ("code", POINTER(c_ubyte)),
-                        ("size", c_int)]
+                        ("size", c_int),
+                        ("get_code", POINTER(c_ubyte)),  # Function ptr
+                        ]
         FrozenTable = POINTER(struct_frozen)
 
         modules = []

Makefile.pre.in

@@ -259,6 +259,7 @@ LIBOBJS=	@LIBOBJS@
 
 PYTHON=		python$(EXE)
 BUILDPYTHON=	python$(BUILDEXE)
+BOOTSTRAP= _bootstrap_python
 
 PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
 UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
@@ -448,6 +449,30 @@ OBJECT_OBJS=	\
 		Objects/unionobject.o \
 		Objects/weakrefobject.o
 
+# DEEPFREEZE_OBJS is auto-generated by Tools/scripts/freeze_modules.py.
+DEEPFREEZE_OBJS = \
+		Python/deepfreeze/importlib._bootstrap.o \
+		Python/deepfreeze/importlib._bootstrap_external.o \
+		Python/deepfreeze/zipimport.o \
+		Python/deepfreeze/abc.o \
+		Python/deepfreeze/codecs.o \
+		Python/deepfreeze/io.o \
+		Python/deepfreeze/_collections_abc.o \
+		Python/deepfreeze/_sitebuiltins.o \
+		Python/deepfreeze/genericpath.o \
+		Python/deepfreeze/ntpath.o \
+		Python/deepfreeze/posixpath.o \
+		Python/deepfreeze/os.o \
+		Python/deepfreeze/site.o \
+		Python/deepfreeze/stat.o \
+		Python/deepfreeze/__hello__.o \
+		Python/deepfreeze/__phello__.o \
+		Python/deepfreeze/__phello__.ham.o \
+		Python/deepfreeze/__phello__.ham.eggs.o \
+		Python/deepfreeze/__phello__.spam.o \
+		Python/deepfreeze/frozen_only.o
+# End DEEPFREEZE_OBJS
+
 ##########################################################################
 # objects that get linked into the Python library
 LIBRARY_OBJS_OMIT_FROZEN=	\
@@ -460,6 +485,7 @@ LIBRARY_OBJS_OMIT_FROZEN=	\
 
 LIBRARY_OBJS=	\
 		$(LIBRARY_OBJS_OMIT_FROZEN) \
+		$(DEEPFREEZE_OBJS) \
 		Python/frozen.o
 
 ##########################################################################
@@ -602,9 +628,9 @@ platform: $(BUILDPYTHON) pybuilddir.txt
 # problems by creating a dummy pybuilddir.txt just to allow interpreter
 # initialization to succeed.  It will be overwritten by generate-posix-vars
 # or removed in case of failure.
-pybuilddir.txt: $(BUILDPYTHON)
+pybuilddir.txt: $(BOOTSTRAP)
 	@echo "none" > ./pybuilddir.txt
-	$(RUNSHARED) $(PYTHON_FOR_BUILD) -S -m sysconfig --generate-posix-vars ;\
+	./$(BOOTSTRAP) -S -m sysconfig --generate-posix-vars ;\
 	if test $$? -ne 0 ; then \
 		echo "generate-posix-vars failed" ; \
 		rm -f ./pybuilddir.txt ; \
@@ -738,6 +764,158 @@ regen-test-frozenmain: $(BUILDPYTHON)
 Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
 	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
 
+############################################################################
+# "Bootstrap Python" used to run deepfreeze.py
+
+BOOTSTRAP_HEADERS = \
+	Python/frozen_modules/importlib._bootstrap.h \
+	Python/frozen_modules/importlib._bootstrap_external.h \
+	Python/frozen_modules/zipimport.h
+
+Python/bootstrap_frozen.o: Python/bootstrap_frozen.c Include/cpython/import.h $(BOOTSTRAP_HEADERS)
+
+$(BOOTSTRAP): $(LIBRARY_OBJS_OMIT_FROZEN) \
+		Python/bootstrap_frozen.o Programs/python.o
+	$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
+		Python/bootstrap_frozen.o \
+		Programs/python.o \
+		$(LIBS) $(MODLIBS) $(SYSLIBS)
+
+############################################################################
+# Deepfreeze targets
+
+.PHONY: regen-deepfreeze
+regen-deepfreeze: $(DEEPFREEZE_OBJS)
+
+DEEPFREEZE_DEPS = \
+	$(BOOTSTRAP) \
+	pybuilddir.txt \
+	$(srcdir)/Tools/scripts/deepfreeze.py
+
+# BEGIN: deepfreeze modules
+
+Python/deepfreeze/importlib._bootstrap.c: $(srcdir)/Lib/importlib/_bootstrap.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/importlib._bootstrap.c from Lib/importlib/_bootstrap.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/importlib/_bootstrap.py -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
+
+Python/deepfreeze/importlib._bootstrap_external.c: $(srcdir)/Lib/importlib/_bootstrap_external.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/importlib._bootstrap_external.c from Lib/importlib/_bootstrap_external.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/importlib/_bootstrap_external.py -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
+
+Python/deepfreeze/zipimport.c: $(srcdir)/Lib/zipimport.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/zipimport.c from Lib/zipimport.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/zipimport.py -m zipimport -o Python/deepfreeze/zipimport.c
+
+Python/deepfreeze/abc.c: $(srcdir)/Lib/abc.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/abc.c from Lib/abc.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/abc.py -m abc -o Python/deepfreeze/abc.c
+
+Python/deepfreeze/codecs.c: $(srcdir)/Lib/codecs.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/codecs.c from Lib/codecs.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/codecs.py -m codecs -o Python/deepfreeze/codecs.c
+
+Python/deepfreeze/io.c: $(srcdir)/Lib/io.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/io.c from Lib/io.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/io.py -m io -o Python/deepfreeze/io.c
+
+Python/deepfreeze/_collections_abc.c: $(srcdir)/Lib/_collections_abc.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/_collections_abc.c from Lib/_collections_abc.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/_collections_abc.py -m _collections_abc -o Python/deepfreeze/_collections_abc.c
+
+Python/deepfreeze/_sitebuiltins.c: $(srcdir)/Lib/_sitebuiltins.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/_sitebuiltins.c from Lib/_sitebuiltins.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/_sitebuiltins.py -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
+
+Python/deepfreeze/genericpath.c: $(srcdir)/Lib/genericpath.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/genericpath.c from Lib/genericpath.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/genericpath.py -m genericpath -o Python/deepfreeze/genericpath.c
+
+Python/deepfreeze/ntpath.c: $(srcdir)/Lib/ntpath.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/ntpath.c from Lib/ntpath.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/ntpath.py -m ntpath -o Python/deepfreeze/ntpath.c
+
+Python/deepfreeze/posixpath.c: $(srcdir)/Lib/posixpath.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/posixpath.c from Lib/posixpath.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/posixpath.py -m posixpath -o Python/deepfreeze/posixpath.c
+
+Python/deepfreeze/os.c: $(srcdir)/Lib/os.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/os.c from Lib/os.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/os.py -m os -o Python/deepfreeze/os.c
+
+Python/deepfreeze/site.c: $(srcdir)/Lib/site.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/site.c from Lib/site.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/site.py -m site -o Python/deepfreeze/site.c
+
+Python/deepfreeze/stat.c: $(srcdir)/Lib/stat.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/stat.c from Lib/stat.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/stat.py -m stat -o Python/deepfreeze/stat.c
+
+Python/deepfreeze/__hello__.c: $(srcdir)/Lib/__hello__.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__hello__.c from Lib/__hello__.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__hello__.py -m __hello__ -o Python/deepfreeze/__hello__.c
+
+Python/deepfreeze/__phello__.c: $(srcdir)/Lib/__phello__/__init__.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.c from Lib/__phello__/__init__.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/__init__.py -m __phello__ -o Python/deepfreeze/__phello__.c
+
+Python/deepfreeze/__phello__.ham.c: $(srcdir)/Lib/__phello__/ham/__init__.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.ham.c from Lib/__phello__/ham/__init__.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/ham/__init__.py -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
+
+Python/deepfreeze/__phello__.ham.eggs.c: $(srcdir)/Lib/__phello__/ham/eggs.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.ham.eggs.c from Lib/__phello__/ham/eggs.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/ham/eggs.py -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
+
+Python/deepfreeze/__phello__.spam.c: $(srcdir)/Lib/__phello__/spam.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/__phello__.spam.c from Lib/__phello__/spam.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Lib/__phello__/spam.py -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
+
+Python/deepfreeze/frozen_only.c: $(srcdir)/Tools/freeze/flag.py $(DEEPFREEZE_DEPS)
+	@echo "Deepfreezing Python/deepfreeze/frozen_only.c from Tools/freeze/flag.py"
+	@./$(BOOTSTRAP) \
+		$(srcdir)/Tools/scripts/deepfreeze.py \
+		$(srcdir)/Tools/freeze/flag.py -m frozen_only -o Python/deepfreeze/frozen_only.c
+
+# END: deepfreeze modules
+
 ############################################################################
 # frozen modules (including importlib)
 
@@ -2017,7 +2195,8 @@ clean-retain-profile: pycremoval
 	find build -name '*.py[co]' -exec rm -f {} ';' || true
 	-rm -f pybuilddir.txt
 	-rm -f Lib/lib2to3/*Grammar*.pickle
-	-rm -f Programs/_testembed Programs/_freeze_module
+	-rm -f Programs/_testembed Programs/_freeze_module $(BOOTSTRAP)
+	-rm -f Python/deepfreeze/*.[co]
 	-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
 	-rm -f Include/pydtrace_probes.h
 	-rm -f profile-gen-stamp

Misc/NEWS.d/next/Build/2021-11-03-00-19-50.bpo-45696.eKs46f.rst

@@ -0,0 +1 @@
+Skip the marshal step for frozen modules by generating C code that produces a set of ready-to-use code objects. This speeds up startup time by another 10% or more.

Python/bootstrap_frozen.c

@@ -0,0 +1,45 @@
+
+/* Frozen modules bootstrap */
+
+/* This file is linked with "bootstrap Python"
+   which is used (only) to run Tools/scripts/deepfreeze.py. */
+
+#include "Python.h"
+#include "pycore_import.h"
+
+/* Includes for frozen modules: */
+#include "frozen_modules/importlib._bootstrap.h"
+#include "frozen_modules/importlib._bootstrap_external.h"
+#include "frozen_modules/zipimport.h"
+/* End includes */
+
+/* Note that a negative size indicates a package. */
+
+static const struct _frozen bootstrap_modules[] = {
+    {"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
+    {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
+    {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
+    {0, 0, 0} /* bootstrap sentinel */
+};
+static const struct _frozen stdlib_modules[] = {
+    {0, 0, 0} /* stdlib sentinel */
+};
+static const struct _frozen test_modules[] = {
+    {0, 0, 0} /* test sentinel */
+};
+const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
+const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
+const struct _frozen *_PyImport_FrozenTest = test_modules;
+
+static const struct _module_alias aliases[] = {
+    {"_frozen_importlib", "importlib._bootstrap"},
+    {"_frozen_importlib_external", "importlib._bootstrap_external"},
+    {0, 0} /* aliases sentinel */
+};
+const struct _module_alias *_PyImport_FrozenAliases = aliases;
+
+
+/* Embedding apps may change this pointer to point to their favorite
+   collection of frozen modules: */
+
+const struct _frozen *PyImport_FrozenModules = NULL;

Python/deepfreeze/README.txt

@@ -0,0 +1,6 @@
+This directory contains the generated .c files for all the deep-frozen
+modules.  Python/frozen.c depends on these files.
+
+None of these files are committed into the repo.
+
+See Tools/scripts/freeze_modules.py for more info.