bpo-45020: Add -X frozen_modules=[on|off] to explicitly control use of frozen modules.

Doc/using/cmdline.rst

@@ -480,6 +480,12 @@ Miscellaneous options
      objects and pyc files are desired as well as supressing the extra visual
      location indicators when the interpreter displays tracebacks. See also
      :envvar:`PYTHONNODEBUGRANGES`.
+   * ``-X frozen_modules`` determines whether or not frozen modules are
+     ignored by the import machinery.  A value of "on" means they get
+     imported and "off" means they are ignored.  The default is "on"
+     for non-debug builds (the normal case) and "off" for debug builds.
+     Note that the "importlib_bootstrap" and "importlib_bootstrap_external"
+     frozen modules are always used, even if this flag is set to "off".
 
    It also allows passing arbitrary values and retrieving them through the
    :data:`sys._xoptions` dictionary.
@@ -518,6 +524,9 @@ Miscellaneous options
    .. versionadded:: 3.11
       The ``-X no_debug_ranges`` option.
 
+   .. versionadded:: 3.11
+      The ``-X frozen_modules`` option.
+
 
 Options you shouldn't use
 ~~~~~~~~~~~~~~~~~~~~~~~~~

Include/cpython/initconfig.h

@@ -2,6 +2,8 @@
 #define Py_PYCORECONFIG_H
 #ifndef Py_LIMITED_API
 
+#include <stdbool.h>
+
 /* --- PyStatus ----------------------------------------------- */
 
 typedef struct {
@@ -172,6 +174,7 @@ typedef struct PyConfig {
     int legacy_windows_stdio;
 #endif
     wchar_t *check_hash_pycs_mode;
+    bool use_frozen_modules;
 
     /* --- Path configuration inputs ------------ */
     int pathconfig_warnings;

Include/internal/pycore_initconfig.h

@@ -155,6 +155,7 @@ extern PyStatus _PyConfig_Copy(
 extern PyStatus _PyConfig_InitPathConfig(
     PyConfig *config,
     int compute_path_config);
+extern PyStatus _PyConfig_InitImportConfig(PyConfig *config);
 extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);

Include/internal/pycore_pystate.h

@@ -30,6 +30,17 @@ _Py_IsMainInterpreter(PyInterpreterState *interp)
 }
 
 
+static inline const PyConfig *
+_Py_GetMainConfig(void)
+{
+    PyInterpreterState *interp = _PyRuntime.interpreters.main;
+    if (interp == NULL) {
+        return NULL;
+    }
+    return _PyInterpreterState_GetConfig(interp);
+}
+
+
 /* Only handle signals on the main thread of the main interpreter. */
 static inline int
 _Py_ThreadCanHandleSignals(PyInterpreterState *interp)

Lib/ctypes/test/test_values.py

@@ -72,14 +72,16 @@ class struct_frozen(Structure):
                 self.assertGreater(abs(entry.size), 10)
                 self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
                 # Check the module's package-ness.
-                spec = importlib.util.find_spec(modname)
+                with import_helper.frozen_modules():
+                    spec = importlib.util.find_spec(modname)
                 if entry.size < 0:
                     # It's a package.
                     self.assertIsNotNone(spec.submodule_search_locations)
                 else:
                     self.assertIsNone(spec.submodule_search_locations)
 
-        expected = imp._frozen_module_names()
+        with import_helper.frozen_modules():
+            expected = imp._frozen_module_names()
         self.maxDiff = None
         self.assertEqual(modules, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")

Lib/test/support/import_helper.py

@@ -109,7 +109,25 @@ def _save_and_block_module(name, orig_modules):
     return saved
 
 
-def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
+@contextlib.contextmanager
+def frozen_modules(enabled=True):
+    """Force frozen modules to be used (or not).
+
+    This only applies to modules that haven't been imported yet.
+    Also, some essential modules will always be imported frozen.
+    """
+    # FYI: the env var will never show up in os.environ.
+    os.putenv('_PYTHONTESTFROZENMODULES', '1' if enabled else '0')
+    try:
+        yield
+    finally:
+        os.unsetenv('_PYTHONTESTFROZENMODULES')
+
+
+def import_fresh_module(name, fresh=(), blocked=(), *,
+                        deprecated=False,
+                        usefrozen=False,
+                        ):
     """Import and return a module, deliberately bypassing sys.modules.
 
     This function imports and returns a fresh copy of the named Python module
@@ -133,6 +151,11 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
 
     This function will raise ImportError if the named module cannot be
     imported.
+
+    If "usefrozen" is False (the default), any stdlib source module will
+    always be imported from its .py file, even if the module has been
+    frozen.  The only exception is essential modules (like
+    importlib._bootstrap).
     """
     # NOTE: test_heapq, test_json and test_warnings include extra sanity checks
     # to make sure that this utility function is working as expected
@@ -148,7 +171,8 @@ def import_fresh_module(name, fresh=(), blocked=(), deprecated=False):
             for blocked_name in blocked:
                 if not _save_and_block_module(blocked_name, orig_modules):
                     names_to_remove.append(blocked_name)
-            fresh_module = importlib.import_module(name)
+            with frozen_modules(usefrozen):
+                fresh_module = importlib.import_module(name)
         except ImportError:
             fresh_module = None
         finally:
@@ -169,9 +193,14 @@ class CleanImport(object):
 
         with CleanImport("foo"):
             importlib.import_module("foo") # new reference
+
+    If "usefrozen" is False (the default), any stdlib source module will
+    always be imported from its .py file, even if the module has been
+    frozen.  The only exception is essential modules (like
+    importlib._bootstrap).
     """
 
-    def __init__(self, *module_names):
+    def __init__(self, *module_names, usefrozen=False):
         self.original_modules = sys.modules.copy()
         for module_name in module_names:
             if module_name in sys.modules:
@@ -183,12 +212,15 @@ def __init__(self, *module_names):
                 if module.__name__ != module_name:
                     del sys.modules[module.__name__]
                 del sys.modules[module_name]
+        self._frozen_modules = frozen_modules(usefrozen)
 
     def __enter__(self):
+        self._frozen_modules.__enter__()
         return self
 
     def __exit__(self, *ignore_exc):
         sys.modules.update(self.original_modules)
+        self._frozen_modules.__exit__(*ignore_exc)
 
 
 class DirsOnSysPath(object):

Lib/test/support/os_helper.py

@@ -599,6 +599,10 @@ def set(self, envvar, value):
     def unset(self, envvar):
         del self[envvar]
 
+    def copy(self):
+        # We do what os.environ.copy() does.
+        return dict(self)
+
     def __enter__(self):
         return self
 

Lib/test/test_embed.py

@@ -12,6 +12,7 @@
 import shutil
 import subprocess
 import sys
+import sysconfig
 import tempfile
 import textwrap
 
@@ -426,11 +427,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pathconfig_warnings': 1,
         '_init_main': 1,
         '_isolated_interpreter': 0,
+        'use_frozen_modules': False,
     }
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
             'legacy_windows_stdio': 0,
         })
+        if not sys.executable.endswith('_d.exe'):
+            # It's not a debug build.
+            CONFIG_COMPAT['use_frozen_modules'] = True
+    else:
+        config_args = sysconfig.get_config_var('CONFIG_ARGS') or ''
+        if '--with-pydebug' not in config_args:
+            CONFIG_COMPAT['use_frozen_modules'] = True
 
     CONFIG_PYTHON = dict(CONFIG_COMPAT,
         _config_init=API_PYTHON,
@@ -1244,7 +1253,9 @@ def test_init_setpythonhome(self):
             'pythonpath_env': paths_str,
         }
         self.default_program_name(config)
-        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
+        env = {'TESTHOME': home,
+               'PYTHONPATH': paths_str,
+               '_PYTHONTESTFROZENMODULES': '1'}
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
 

Lib/test/test_frozen.py

@@ -12,16 +12,17 @@
 
 import sys
 import unittest
-from test.support import captured_stdout
+from test.support import captured_stdout, import_helper
 
 
 class TestFrozen(unittest.TestCase):
     def test_frozen(self):
         name = '__hello__'
         if name in sys.modules:
             del sys.modules[name]
-        with captured_stdout() as out:
-            import __hello__
+        with import_helper.frozen_modules():
+            with captured_stdout() as out:
+                import __hello__
         self.assertEqual(out.getvalue(), 'Hello world!\n')
 
 

Lib/test/test_importlib/frozen/test_finder.py

@@ -6,14 +6,17 @@
 import unittest
 import warnings
 
+from test.support import import_helper
+
 
 class FindSpecTests(abc.FinderTests):
 
     """Test finding frozen modules."""
 
     def find(self, name, path=None):
         finder = self.machinery.FrozenImporter
-        return finder.find_spec(name, path)
+        with import_helper.frozen_modules():
+            return finder.find_spec(name, path)
 
     def test_module(self):
         name = '__hello__'
@@ -52,7 +55,8 @@ def find(self, name, path=None):
         finder = self.machinery.FrozenImporter
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", DeprecationWarning)
-            return finder.find_module(name, path)
+            with import_helper.frozen_modules():
+                return finder.find_module(name, path)
 
     def test_module(self):
         name = '__hello__'