bpo-32030: Split Py_Main() into subfunctions (python#4399)

* Don't use "Python runtime" anymore to parse command line options or
  to get environment variables: pymain_init() is now a strict
  separation.
* Use an error message rather than "crashing" directly with
  Py_FatalError(). Limit the number of calls to Py_FatalError(). It
  prepares the code to handle errors more nicely later.
* Warnings options (-W, PYTHONWARNINGS) and "XOptions" (-X) are now
  only added to the sys module once Python core is properly
  initialized.
* _PyMain is now the well identified owner of some important strings
  like: warnings options, XOptions, and the "program name". The
  program name string is now properly freed at exit.
  pymain_free() is now responsible to free the "command" string.
* Rename most methods in Modules/main.c to use a "pymain_" prefix to
  avoid conflits and ease debug.
* Replace _Py_CommandLineDetails_INIT with memset(0)
* Reorder a lot of code to fix the initialization ordering. For
  example, initializing standard streams now comes before parsing
  PYTHONWARNINGS.
* Py_Main() now handles errors when adding warnings options and
  XOptions.
* Add _PyMem_GetDefaultRawAllocator() private function.
* Cleanup _PyMem_Initialize(): remove useless global constants: move
  them into _PyMem_Initialize().
* Call _PyRuntime_Initialize() as soon as possible:
  _PyRuntime_Initialize() now returns an error message on failure.
* Add _PyInitError structure and following macros:

  * _Py_INIT_OK()
  * _Py_INIT_ERR(msg)
  * _Py_INIT_USER_ERR(msg): "user" error, don't abort() in that case
  * _Py_INIT_FAILED(err)

Include/import.h

@@ -8,7 +8,7 @@ extern "C" {
 #endif
 
 #ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _PyImportZip_Init(void);
+PyAPI_FUNC(_PyInitError) _PyImportZip_Init(void);
 
 PyMODINIT_FUNC PyInit_imp(void);
 #endif /* !Py_LIMITED_API */

Include/internal/pystate.h

@@ -74,10 +74,16 @@ typedef struct pyruntimestate {
     // XXX Consolidate globals found via the check-c-globals script.
 } _PyRuntimeState;
 
+#define _PyRuntimeState_INIT {.initialized = 0, .core_initialized = 0}
+
 PyAPI_DATA(_PyRuntimeState) _PyRuntime;
-PyAPI_FUNC(void) _PyRuntimeState_Init(_PyRuntimeState *);
+PyAPI_FUNC(_PyInitError) _PyRuntimeState_Init(_PyRuntimeState *);
 PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *);
 
+/* Initialize _PyRuntimeState.
+   Return NULL on success, or return an error message on failure. */
+PyAPI_FUNC(_PyInitError) _PyRuntime_Initialize(void);
+
 #define _Py_CURRENTLY_FINALIZING(tstate) \
     (_PyRuntime.finalizing == tstate)
 

Include/pylifecycle.h

@@ -20,18 +20,44 @@ PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
 PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
                                              const char *errors);
 
+typedef struct {
+    const char *prefix;
+    const char *msg;
+    int user_err;
+} _PyInitError;
+
+/* Almost all errors causing Python initialization to fail */
+#ifdef _MSC_VER
+   /* Visual Studio 2015 doesn't implement C99 __func__ in C */
+#  define _Py_INIT_GET_FUNC() __FUNCTION__
+#else
+#  define _Py_INIT_GET_FUNC() __func__
+#endif
+
+#define _Py_INIT_OK() \
+    (_PyInitError){.prefix = NULL, .msg = NULL, .user_err = 0}
+#define _Py_INIT_ERR(MSG) \
+    (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 0}
+/* Error that can be fixed by the user like invalid input parameter.
+   Don't abort() the process on such error. */
+#define _Py_INIT_USER_ERR(MSG) \
+    (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1}
+#define _Py_INIT_FAILED(err) \
+    (err.msg != NULL)
+
 /* PEP 432 Multi-phase initialization API (Private while provisional!) */
-PyAPI_FUNC(void) _Py_InitializeCore(const _PyCoreConfig *);
+PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *);
 PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
-PyAPI_FUNC(int) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *);
-PyAPI_FUNC(int) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
+PyAPI_FUNC(_PyInitError) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *);
+PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
 #endif
 
 /* Initialization and finalization */
 PyAPI_FUNC(void) Py_Initialize(void);
 PyAPI_FUNC(void) Py_InitializeEx(int);
 #ifndef Py_LIMITED_API
-PyAPI_FUNC(void) _Py_InitializeEx_Private(int, int);
+PyAPI_FUNC(_PyInitError) _Py_InitializeEx_Private(int, int);
+PyAPI_FUNC(void) _Py_FatalInitError(_PyInitError err) _Py_NO_RETURN;
 #endif
 PyAPI_FUNC(void) Py_Finalize(void);
 PyAPI_FUNC(int) Py_FinalizeEx(void);
@@ -50,7 +76,7 @@ PyAPI_FUNC(void) _Py_PyAtExit(void (*func)(void));
 #endif
 PyAPI_FUNC(int) Py_AtExit(void (*func)(void));
 
-PyAPI_FUNC(void) Py_Exit(int);
+PyAPI_FUNC(void) Py_Exit(int) _Py_NO_RETURN;
 
 /* Restore signals that the interpreter has called SIG_IGN on to SIG_DFL. */
 #ifndef Py_LIMITED_API
@@ -86,15 +112,15 @@ PyAPI_FUNC(const char *) _Py_gitversion(void);
 /* Internal -- various one-time initializations */
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void);
-PyAPI_FUNC(PyObject *) _PySys_BeginInit(void);
+PyAPI_FUNC(_PyInitError) _PySys_BeginInit(PyObject **sysmod);
 PyAPI_FUNC(int) _PySys_EndInit(PyObject *sysdict);
-PyAPI_FUNC(void) _PyImport_Init(void);
+PyAPI_FUNC(_PyInitError) _PyImport_Init(void);
 PyAPI_FUNC(void) _PyExc_Init(PyObject * bltinmod);
-PyAPI_FUNC(void) _PyImportHooks_Init(void);
+PyAPI_FUNC(_PyInitError) _PyImportHooks_Init(void);
 PyAPI_FUNC(int) _PyFrame_Init(void);
 PyAPI_FUNC(int) _PyFloat_Init(void);
 PyAPI_FUNC(int) PyByteArray_Init(void);
-PyAPI_FUNC(void) _Py_HashRandomization_Init(_PyCoreConfig *core_config);
+PyAPI_FUNC(_PyInitError) _Py_HashRandomization_Init(_PyCoreConfig *core_config);
 #endif
 
 /* Various internal finalizers */

Include/pymem.h

@@ -223,6 +223,10 @@ PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain,
 PyAPI_FUNC(void) PyMem_SetupDebugHooks(void);
 #endif
 
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(void) _PyMem_GetDefaultRawAllocator(PyMemAllocatorEx *alloc);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Include/sysmodule.h

@@ -37,6 +37,11 @@ PyAPI_FUNC(PyObject *) PySys_GetXOptions(void);
 PyAPI_FUNC(size_t) _PySys_GetSizeOf(PyObject *);
 #endif
 
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(int) _PySys_AddXOptionWithError(const wchar_t *s);
+PyAPI_FUNC(int) _PySys_AddWarnOptionWithError(PyObject *option);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

Modules/faulthandler.c

@@ -1281,47 +1281,54 @@ PyInit_faulthandler(void)
     return m;
 }
 
-/* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable
-   is defined, or if sys._xoptions has a 'faulthandler' key. */
-
 static int
-faulthandler_env_options(void)
+faulthandler_init_enable(void)
 {
-    PyObject *xoptions, *key, *module, *res;
-    char *p;
+    PyObject *module = PyImport_ImportModule("faulthandler");
+    if (module == NULL) {
+        return -1;
+    }
 
-    if (!((p = Py_GETENV("PYTHONFAULTHANDLER")) && *p != '\0')) {
-        /* PYTHONFAULTHANDLER environment variable is missing
-           or an empty string */
-        int has_key;
+    PyObject *res = _PyObject_CallMethodId(module, &PyId_enable, NULL);
+    Py_DECREF(module);
+    if (res == NULL) {
+        return -1;
+    }
+    Py_DECREF(res);
 
-        xoptions = PySys_GetXOptions();
-        if (xoptions == NULL)
-            return -1;
+    return 0;
+}
 
-        key = PyUnicode_FromString("faulthandler");
-        if (key == NULL)
-            return -1;
+/* Call faulthandler.enable() if the PYTHONFAULTHANDLER environment variable
+   is defined, or if sys._xoptions has a 'faulthandler' key. */
 
-        has_key = PyDict_Contains(xoptions, key);
-        Py_DECREF(key);
-        if (has_key <= 0)
-            return has_key;
+static int
+faulthandler_init_parse(void)
+{
+    char *p = Py_GETENV("PYTHONFAULTHANDLER");
+    if (p && *p != '\0') {
+        return 1;
     }
 
-    module = PyImport_ImportModule("faulthandler");
-    if (module == NULL) {
+    /* PYTHONFAULTHANDLER environment variable is missing
+       or an empty string */
+    PyObject *xoptions = PySys_GetXOptions();
+    if (xoptions == NULL) {
         return -1;
     }
-    res = _PyObject_CallMethodId(module, &PyId_enable, NULL);
-    Py_DECREF(module);
-    if (res == NULL)
+
+    PyObject *key = PyUnicode_FromString("faulthandler");
+    if (key == NULL) {
         return -1;
-    Py_DECREF(res);
-    return 0;
+    }
+
+    int has_key = PyDict_Contains(xoptions, key);
+    Py_DECREF(key);
+    return has_key;
 }
 
-int _PyFaulthandler_Init(void)
+_PyInitError
+_PyFaulthandler_Init(void)
 {
 #ifdef HAVE_SIGALTSTACK
     int err;
@@ -1345,14 +1352,22 @@ int _PyFaulthandler_Init(void)
     thread.cancel_event = PyThread_allocate_lock();
     thread.running = PyThread_allocate_lock();
     if (!thread.cancel_event || !thread.running) {
-        PyErr_SetString(PyExc_RuntimeError,
-                        "could not allocate locks for faulthandler");
-        return -1;
+        return _Py_INIT_ERR("failed to allocate locks for faulthandler");
     }
     PyThread_acquire_lock(thread.cancel_event, 1);
 #endif
 
-    return faulthandler_env_options();
+    int enable = faulthandler_init_parse();
+    if (enable < 0) {
+        return _Py_INIT_ERR("failed to parse faulthandler env var and cmdline");
+    }
+
+    if (enable) {
+        if (faulthandler_init_enable() < 0) {
+            return _Py_INIT_ERR("failed to enable faulthandler");
+        }
+    }
+    return _Py_INIT_OK();
 }
 
 void _PyFaulthandler_Fini(void)