Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

gh-108716: Cleanup remaining deepfreeze infrastructure #116919

Merged
merged 2 commits into from
Mar 18, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Clean up remaining deepfreeze infrastructure
Keep Tools/build/deepfreeze.py, we may repurpose it in the future.
Put a line in find . -depth -name '__pycache__' -exec rm -rf {} ';'
find . -name '*.py[co]' -exec rm -f {} ';'
find . -name '*.[oa]' -exec rm -f {} ';'
find . -name '*.s[ol]' -exec rm -f {} ';'
find . -name '*.so.[0-9]*.[0-9]*' -exec rm -f {} ';'
find . -name '*.lto' -exec rm -f {} ';'
find . -name '*.wasm' -exec rm -f {} ';'
find . -name '*.lst' -exec rm -f {} ';'
find build -name 'fficonfig.h' -exec rm -f {} ';' || true
find build -name '*.py' -exec rm -f {} ';' || true
find build -name '*.py[co]' -exec rm -f {} ';' || true
rm -f pybuilddir.txt
rm -f _bootstrap_python
rm -f python.html python*.js python.data python*.symbols python*.map
rm -f ./usr/local/lib/python3.13/os.py
rm -f Programs/_testembed Programs/_freeze_module
rm -rf Python/deepfreeze
rm -f Python/frozen_modules/*.h
rm -f Python/frozen_modules/MANIFEST
rm -f jit_stencils.h
find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
rm -f Include/pydtrace_probes.h
rm -f profile-gen-stamp
rm -rf iOS/testbed/Python.xcframework/ios-*/bin
rm -rf iOS/testbed/Python.xcframework/ios-*/lib
rm -rf iOS/testbed/Python.xcframework/ios-*/include
rm -rf iOS/testbed/Python.xcframework/ios-*/Python.framework
# Profile data.
rm -f *.fdata
# Pristine binaries before BOLT optimization.
rm -f *.prebolt
# BOLT instrumented binaries.
rm -f *.bolt_inst that removes Python/deepfreeze, for current devs.

TODO: Windows project files
  • Loading branch information
gvanrossum committed Mar 16, 2024
commit 61284cbad815124dc9b9c28c92d9166d7562c5f4
1 change: 0 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,6 @@ Tools/unicode/data/
/profile-clean-stamp
/profile-run-stamp
/profile-bolt-stamp
/Python/deepfreeze/*.c
/pybuilddir.txt
/pyconfig.h
/python-config
Expand Down
43 changes: 3 additions & 40 deletions Makefile.pre.in
Original file line number Diff line number Diff line change
Expand Up @@ -1400,7 +1400,7 @@ Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS)
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)

############################################################################
# "Bootstrap Python" used to run deepfreeze.py
# "Bootstrap Python" used to run Programs/_freeze_module.py

BOOTSTRAP_HEADERS = \
Python/frozen_modules/importlib._bootstrap.h \
Expand All @@ -1419,7 +1419,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
#
# Freezing is a multi step process. It works differently for standard builds
# and cross builds. Standard builds use Programs/_freeze_module and
# _bootstrap_python for freezing and deepfreezing, so users can build Python
# _bootstrap_python for freezing, so users can build Python
# without an existing Python installation. Cross builds cannot execute
# compiled binaries and therefore rely on an external build Python
# interpreter. The build interpreter must have same version and same bytecode
Expand All @@ -1433,12 +1433,10 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
# 5) create remaining frozen module headers with
# ``./_bootstrap_python Programs/_freeze_module.py``. The pure Python
# script is used to test the cross compile code path.
# 6) deepfreeze modules with _bootstrap_python
#
# Cross compile process:
# 1) create all frozen module headers with external build Python and
# Programs/_freeze_module.py script.
# 2) deepfreeze modules with external build Python.
#

# FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py.
Expand Down Expand Up @@ -1584,41 +1582,6 @@ regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN)
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py --frozen-modules
@echo "The Makefile was updated, you may need to re-run make."

############################################################################
# Deepfreeze targets

DEEPFREEZE_C = Python/deepfreeze/deepfreeze.c
DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py Include/internal/pycore_global_strings.h $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT)

# BEGIN: deepfreeze modules
$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \
Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \
Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \
Python/frozen_modules/zipimport.h:zipimport \
Python/frozen_modules/abc.h:abc \
Python/frozen_modules/codecs.h:codecs \
Python/frozen_modules/io.h:io \
Python/frozen_modules/_collections_abc.h:_collections_abc \
Python/frozen_modules/_sitebuiltins.h:_sitebuiltins \
Python/frozen_modules/genericpath.h:genericpath \
Python/frozen_modules/ntpath.h:ntpath \
Python/frozen_modules/posixpath.h:posixpath \
Python/frozen_modules/os.h:os \
Python/frozen_modules/site.h:site \
Python/frozen_modules/stat.h:stat \
Python/frozen_modules/importlib.util.h:importlib.util \
Python/frozen_modules/importlib.machinery.h:importlib.machinery \
Python/frozen_modules/runpy.h:runpy \
Python/frozen_modules/__hello__.h:__hello__ \
Python/frozen_modules/__phello__.h:__phello__ \
Python/frozen_modules/__phello__.ham.h:__phello__.ham \
Python/frozen_modules/__phello__.ham.eggs.h:__phello__.ham.eggs \
Python/frozen_modules/__phello__.spam.h:__phello__.spam \
Python/frozen_modules/frozen_only.h:frozen_only \
-o Python/deepfreeze/deepfreeze.c
# END: deepfreeze modules

# We keep this renamed target around for folks with muscle memory.
.PHONY: regen-importlib
regen-importlib: regen-frozen
Expand Down Expand Up @@ -2933,7 +2896,7 @@ clean-retain-profile: pycremoval
-rm -f python.html python*.js python.data python*.symbols python*.map
-rm -f $(WASM_STDLIB)
-rm -f Programs/_testembed Programs/_freeze_module
-rm -f Python/deepfreeze/*.[co]
-rm -rf Python/deepfreeze
-rm -f Python/frozen_modules/*.h
-rm -f Python/frozen_modules/MANIFEST
-rm -f jit_stencils.h
Expand Down
6 changes: 0 additions & 6 deletions Python/deepfreeze/README.txt

This file was deleted.

28 changes: 0 additions & 28 deletions Python/frozen.c
Original file line number Diff line number Diff line change
Expand Up @@ -66,34 +66,6 @@
#include "frozen_modules/frozen_only.h"
/* End includes */

#define GET_CODE(name) _Py_get_##name##_toplevel

/* Start extern declarations */
extern PyObject *_Py_get_importlib__bootstrap_toplevel(void);
extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void);
extern PyObject *_Py_get_zipimport_toplevel(void);
extern PyObject *_Py_get_abc_toplevel(void);
extern PyObject *_Py_get_codecs_toplevel(void);
extern PyObject *_Py_get_io_toplevel(void);
extern PyObject *_Py_get__collections_abc_toplevel(void);
extern PyObject *_Py_get__sitebuiltins_toplevel(void);
extern PyObject *_Py_get_genericpath_toplevel(void);
extern PyObject *_Py_get_ntpath_toplevel(void);
extern PyObject *_Py_get_posixpath_toplevel(void);
extern PyObject *_Py_get_os_toplevel(void);
extern PyObject *_Py_get_site_toplevel(void);
extern PyObject *_Py_get_stat_toplevel(void);
extern PyObject *_Py_get_importlib_util_toplevel(void);
extern PyObject *_Py_get_importlib_machinery_toplevel(void);
extern PyObject *_Py_get_runpy_toplevel(void);
extern PyObject *_Py_get___hello___toplevel(void);
extern PyObject *_Py_get___phello___toplevel(void);
extern PyObject *_Py_get___phello___ham_toplevel(void);
extern PyObject *_Py_get___phello___ham_eggs_toplevel(void);
extern PyObject *_Py_get___phello___spam_toplevel(void);
extern PyObject *_Py_get_frozen_only_toplevel(void);
/* End extern declarations */

static const struct _frozen bootstrap_modules[] = {
{"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap), false},
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external), false},
Expand Down
79 changes: 2 additions & 77 deletions Tools/build/freeze_modules.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@
# If FROZEN_MODULES_DIR or DEEPFROZEN_MODULES_DIR is changed then the
# .gitattributes and .gitignore files needs to be updated.
FROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'frozen_modules')
DEEPFROZEN_MODULES_DIR = os.path.join(ROOT_DIR, 'Python', 'deepfreeze')
DEEPFREEZE_MAPPING_FNAME = 'deepfreeze_mappings.txt'

FROZEN_FILE = os.path.join(ROOT_DIR, 'Python', 'frozen.c')
MAKEFILE = os.path.join(ROOT_DIR, 'Makefile.pre.in')
Expand Down Expand Up @@ -233,16 +231,15 @@ def iter_subs():
#######################################
# frozen source files

class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile deepfreezefile')):
class FrozenSource(namedtuple('FrozenSource', 'id pyfile frozenfile')):

@classmethod
def from_id(cls, frozenid, pyfile=None):
if not pyfile:
pyfile = os.path.join(STDLIB_DIR, *frozenid.split('.')) + '.py'
#assert os.path.exists(pyfile), (frozenid, pyfile)
frozenfile = resolve_frozen_file(frozenid, FROZEN_MODULES_DIR)
deepfreezefile = resolve_frozen_file(frozenid, DEEPFROZEN_MODULES_DIR)
return cls(frozenid, pyfile, frozenfile, deepfreezefile)
return cls(frozenid, pyfile, frozenfile)

@property
def frozenid(self):
Expand Down Expand Up @@ -508,13 +505,6 @@ def regen_frozen(modules):
lines.append(f'/* {mod.section} */')
lastsection = mod.section

# Also add a extern declaration for the corresponding
# deepfreeze-generated function.
orig_name = mod.source.id
code_name = orig_name.replace(".", "_")
get_code_name = "_Py_get_%s_toplevel" % code_name
externlines.append("extern PyObject *%s(void);" % get_code_name)

pkg = 'true' if mod.ispkg else 'false'
size = f"(int)sizeof({mod.symbol})"
line = f'{{"{mod.name}", {mod.symbol}, {size}, {pkg}}},'
Expand Down Expand Up @@ -549,13 +539,6 @@ def regen_frozen(modules):
headerlines,
FROZEN_FILE,
)
lines = replace_block(
lines,
"/* Start extern declarations */",
"/* End extern declarations */",
externlines,
FROZEN_FILE,
)
lines = replace_block(
lines,
"static const struct _frozen bootstrap_modules[] =",
Expand Down Expand Up @@ -591,8 +574,6 @@ def regen_makefile(modules):
pyfiles = []
frozenfiles = []
rules = ['']
deepfreezerules = ["$(DEEPFREEZE_C): $(DEEPFREEZE_DEPS)",
"\t$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \\"]
for src in _iter_sources(modules):
frozen_header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
frozenfiles.append(f'\t\t{frozen_header} \\')
Expand All @@ -614,8 +595,6 @@ def regen_makefile(modules):
f'\t{freeze}',
'',
])
deepfreezerules.append(f"\t{frozen_header}:{src.frozenid} \\")
deepfreezerules.append('\t-o Python/deepfreeze/deepfreeze.c')
pyfiles[-1] = pyfiles[-1].rstrip(" \\")
frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")

Expand Down Expand Up @@ -643,23 +622,13 @@ def regen_makefile(modules):
rules,
MAKEFILE,
)
lines = replace_block(
lines,
"# BEGIN: deepfreeze modules",
"# END: deepfreeze modules",
deepfreezerules,
MAKEFILE,
)
outfile.writelines(lines)


def regen_pcbuild(modules):
projlines = []
filterlines = []
corelines = []
deepfreezemappingsfile = f'$(IntDir)\\{DEEPFREEZE_MAPPING_FNAME}'
deepfreezerules = [f' <Exec Command=\'$(PythonForBuild) "$(PySourcePath)Tools\\build\\deepfreeze.py" -f "{deepfreezemappingsfile}" -o "$(GeneratedFrozenModulesDir)deepfreeze.c"\' />']
deepfreezemappings = []
for src in _iter_sources(modules):
pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR)
header = relpath_for_windows_display(src.frozenfile, ROOT_DIR)
Expand All @@ -673,9 +642,6 @@ def regen_pcbuild(modules):
filterlines.append(f' <None Include="..\\{pyfile}">')
filterlines.append(' <Filter>Python Files</Filter>')
filterlines.append(' </None>')
deepfreezemappings.append(f' <FrozenModule Include="$(GeneratedFrozenModulesDir){header}" FrozenId="{src.frozenid}" />\n')

corelines.append(f' <ClCompile Include="$(GeneratedFrozenModulesDir)deepfreeze.c" />')

print(f'# Updating {os.path.relpath(PCBUILD_PROJECT)}')
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
Expand All @@ -688,36 +654,6 @@ def regen_pcbuild(modules):
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mappings -->',
'<!-- END freeze mappings -->',
deepfreezemappings,
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN freeze mapping file -->',
'<!-- END freeze mapping file -->',
[deepfreezemappingsfile, ],
PCBUILD_PROJECT,
)
outfile.writelines(lines)
with updating_file_with_tmpfile(PCBUILD_PROJECT) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN deepfreeze rule -->',
'<!-- END deepfreeze rule -->',
deepfreezerules,
PCBUILD_PROJECT,
)
outfile.writelines(lines)
print(f'# Updating {os.path.relpath(PCBUILD_FILTERS)}')
with updating_file_with_tmpfile(PCBUILD_FILTERS) as (infile, outfile):
lines = infile.readlines()
Expand All @@ -729,17 +665,6 @@ def regen_pcbuild(modules):
PCBUILD_FILTERS,
)
outfile.writelines(lines)
print(f'# Updating {os.path.relpath(PCBUILD_PYTHONCORE)}')
with updating_file_with_tmpfile(PCBUILD_PYTHONCORE) as (infile, outfile):
lines = infile.readlines()
lines = replace_block(
lines,
'<!-- BEGIN deepfreeze -->',
'<!-- END deepfreeze -->',
corelines,
PCBUILD_FILTERS,
)
outfile.writelines(lines)


#######################################
Expand Down
3 changes: 1 addition & 2 deletions configure

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions configure.ac
Original file line number Diff line number Diff line change
Expand Up @@ -6826,8 +6826,7 @@ SRCDIRS="\
Parser/lexer \
Programs \
Python \
Python/frozen_modules \
Python/deepfreeze"
Python/frozen_modules"
AC_MSG_CHECKING([for build directories])
for dir in $SRCDIRS; do
if test ! -d $dir; then
Expand Down