gh-91985: Ensure in-tree builds override platstdlib_dir in every path calculation (GH-93641)

Author: neonene

Lib/test/test_embed.py

@@ -1303,6 +1303,66 @@ def test_init_setpythonhome(self):
         self.check_all_configs("test_init_setpythonhome", config,
                                api=API_COMPAT, env=env)
 
+    def test_init_is_python_build_with_home(self):
+        # Test _Py_path_config._is_python_build configuration (gh-91985)
+        config = self._get_expected_config()
+        paths = config['config']['module_search_paths']
+        paths_str = os.path.pathsep.join(paths)
+
+        for path in paths:
+            if not os.path.isdir(path):
+                continue
+            if os.path.exists(os.path.join(path, 'os.py')):
+                home = os.path.dirname(path)
+                break
+        else:
+            self.fail(f"Unable to find home in {paths!r}")
+
+        prefix = exec_prefix = home
+        if MS_WINDOWS:
+            stdlib = os.path.join(home, "Lib")
+            # Because we are specifying 'home', module search paths
+            # are fairly static
+            expected_paths = [paths[0], stdlib, os.path.join(home, 'DLLs')]
+        else:
+            version = f'{sys.version_info.major}.{sys.version_info.minor}'
+            stdlib = os.path.join(home, sys.platlibdir, f'python{version}')
+            expected_paths = self.module_search_paths(prefix=home, exec_prefix=home)
+
+        config = {
+            'home': home,
+            'module_search_paths': expected_paths,
+            'prefix': prefix,
+            'base_prefix': prefix,
+            'exec_prefix': exec_prefix,
+            'base_exec_prefix': exec_prefix,
+            'pythonpath_env': paths_str,
+            'stdlib_dir': stdlib,
+        }
+        # The code above is taken from test_init_setpythonhome()
+        env = {'TESTHOME': home, 'PYTHONPATH': paths_str}
+
+        env['NEGATIVE_ISPYTHONBUILD'] = '1'
+        config['_is_python_build'] = 0
+        self.check_all_configs("test_init_is_python_build", config,
+                               api=API_COMPAT, env=env)
+
+        env['NEGATIVE_ISPYTHONBUILD'] = '0'
+        config['_is_python_build'] = 1
+        exedir = os.path.dirname(sys.executable)
+        with open(os.path.join(exedir, 'pybuilddir.txt'), encoding='utf8') as f:
+            expected_paths[2] = os.path.normpath(
+                os.path.join(exedir, f'{f.read()}\n$'.splitlines()[0]))
+        if not MS_WINDOWS:
+            # PREFIX (default) is set when running in build directory
+            prefix = exec_prefix = sys.prefix
+            # stdlib calculation (/Lib) is not yet supported
+            expected_paths[0] = self.module_search_paths(prefix=prefix)[0]
+            config.update(prefix=prefix, base_prefix=prefix,
+                          exec_prefix=exec_prefix, base_exec_prefix=exec_prefix)
+        self.check_all_configs("test_init_is_python_build", config,
+                               api=API_COMPAT, env=env)
+
     def copy_paths_by_env(self, config):
         all_configs = self._get_expected_config()
         paths = all_configs['config']['module_search_paths']

Modules/getpath.py

@@ -461,7 +461,8 @@ def search_up(prefix, *landmarks, test=isfile):
 
 build_prefix = None
 
-if not home_was_set and real_executable_dir and not py_setpath:
+if ((not home_was_set and real_executable_dir and not py_setpath)
+        or config.get('_is_python_build', 0) > 0):
     # Detect a build marker and use it to infer prefix, exec_prefix,
     # stdlib_dir and the platstdlib_dir directories.
     try:

Programs/_testembed.c

@@ -1550,6 +1550,46 @@ static int test_init_setpythonhome(void)
 }
 
 
+static int test_init_is_python_build(void)
+{
+    // gh-91985: in-tree builds fail to check for build directory landmarks
+    // under the effect of 'home' or PYTHONHOME environment variable.
+    char *env = getenv("TESTHOME");
+    if (!env) {
+        error("missing TESTHOME env var");
+        return 1;
+    }
+    wchar_t *home = Py_DecodeLocale(env, NULL);
+    if (home == NULL) {
+        error("failed to decode TESTHOME");
+        return 1;
+    }
+
+    PyConfig config;
+    _PyConfig_InitCompatConfig(&config);
+    config_set_program_name(&config);
+    config_set_string(&config, &config.home, home);
+    PyMem_RawFree(home);
+    putenv("TESTHOME=");
+
+    // Use an impossible value so we can detect whether it isn't updated
+    // during initialization.
+    config._is_python_build = INT_MAX;
+    env = getenv("NEGATIVE_ISPYTHONBUILD");
+    if (env && strcmp(env, "0") != 0) {
+        config._is_python_build++;
+    }
+    init_from_config_clear(&config);
+    Py_Finalize();
+    // Second initialization
+    config._is_python_build = -1;
+    init_from_config_clear(&config);
+    dump_config();  // home and _is_python_build are cached in _Py_path_config
+    Py_Finalize();
+    return 0;
+}
+
+
 static int test_init_warnoptions(void)
 {
     putenv("PYTHONWARNINGS=ignore:::env1,ignore:::env2");
@@ -1965,6 +2005,7 @@ static struct TestCase TestCases[] = {
     {"test_init_setpath", test_init_setpath},
     {"test_init_setpath_config", test_init_setpath_config},
     {"test_init_setpythonhome", test_init_setpythonhome},
+    {"test_init_is_python_build", test_init_is_python_build},
     {"test_init_warnoptions", test_init_warnoptions},
     {"test_init_set_config", test_init_set_config},
     {"test_run_main", test_run_main},

Python/pathconfig.c

@@ -36,10 +36,11 @@ typedef struct _PyPathConfig {
     wchar_t *program_name;
     /* Set by Py_SetPythonHome() or PYTHONHOME environment variable */
     wchar_t *home;
+    int _is_python_build;
 } _PyPathConfig;
 
 #  define _PyPathConfig_INIT \
-      {.module_search_path = NULL}
+      {.module_search_path = NULL, ._is_python_build = 0}
 
 
 _PyPathConfig _Py_path_config = _PyPathConfig_INIT;
@@ -72,6 +73,7 @@ _PyPathConfig_ClearGlobal(void)
     CLEAR(calculated_module_search_path);
     CLEAR(program_name);
     CLEAR(home);
+    _Py_path_config._is_python_build = 0;
 
 #undef CLEAR
 
@@ -99,15 +101,25 @@ _PyPathConfig_ReadGlobal(PyConfig *config)
         } \
     } while (0)
 
+#define COPY_INT(ATTR) \
+    do { \
+        assert(_Py_path_config.ATTR >= 0); \
+        if ((_Py_path_config.ATTR >= 0) && (config->ATTR <= 0)) { \
+            config->ATTR = _Py_path_config.ATTR; \
+        } \
+    } while (0)
+
     COPY(prefix);
     COPY(exec_prefix);
     COPY(stdlib_dir);
     COPY(program_name);
     COPY(home);
     COPY2(executable, program_full_path);
+    COPY_INT(_is_python_build);
     // module_search_path must be initialised - not read
 #undef COPY
 #undef COPY2
+#undef COPY_INT
 
 done:
     return status;
@@ -137,14 +149,23 @@ _PyPathConfig_UpdateGlobal(const PyConfig *config)
         } \
     } while (0)
 
+#define COPY_INT(ATTR) \
+    do { \
+        if (config->ATTR > 0) { \
+            _Py_path_config.ATTR = config->ATTR; \
+        } \
+    } while (0)
+
     COPY(prefix);
     COPY(exec_prefix);
     COPY(stdlib_dir);
     COPY(program_name);
     COPY(home);
     COPY2(program_full_path, executable);
+    COPY_INT(_is_python_build);
 #undef COPY
 #undef COPY2
+#undef COPY_INT
 
     PyMem_RawFree(_Py_path_config.module_search_path);
     _Py_path_config.module_search_path = NULL;