bpo-36775: _PyCoreConfig only uses wchar_t* (pythonGH-13062)

_PyCoreConfig: Change filesystem_encoding, filesystem_errors,
stdio_encoding and stdio_errors fields type from char* to wchar_t*.

Changes:

* PyInterpreterState: replace fscodec_initialized (int) with fs_codec
  structure.
* Add get_error_handler_wide() and unicode_encode_utf8() helper
  functions.
* Add error_handler parameter to unicode_encode_locale()
  and unicode_decode_locale().
* Remove _PyCoreConfig_SetString().
* Rename _PyCoreConfig_SetWideString() to _PyCoreConfig_SetString().
* Rename _PyCoreConfig_SetWideStringFromString()
  to _PyCoreConfig_DecodeLocale().

Author: vstinner

Include/cpython/coreconfig.h

@@ -207,8 +207,8 @@ typedef struct {
 
        See Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors.
        */
-    char *filesystem_encoding;
-    char *filesystem_errors;
+    wchar_t *filesystem_encoding;
+    wchar_t *filesystem_errors;
 
     wchar_t *pycache_prefix;  /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */
     wchar_t *program_name;    /* Program name, see also Py_GetProgramName() */
@@ -334,13 +334,13 @@ typedef struct {
        Value set from PYTHONIOENCODING environment variable and
        Py_SetStandardStreamEncoding() function.
        See also 'stdio_errors' attribute. */
-    char *stdio_encoding;
+    wchar_t *stdio_encoding;
 
     /* Error handler of sys.stdin and sys.stdout.
        Value set from PYTHONIOENCODING environment variable and
        Py_SetStandardStreamEncoding() function.
        See also 'stdio_encoding' attribute. */
-    char *stdio_errors;
+    wchar_t *stdio_errors;
 
 #ifdef MS_WINDOWS
     /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys

Include/internal/pycore_coreconfig.h

@@ -106,12 +106,9 @@ PyAPI_FUNC(_PyInitError) _PyCoreConfig_Copy(
     _PyCoreConfig *config,
     const _PyCoreConfig *config2);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetString(
-    char **config_str,
-    const char *str);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetWideString(
     wchar_t **config_str,
     const wchar_t *str);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetWideStringFromString(
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_DecodeLocale(
     wchar_t **config_str,
     const char *str);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config);

Include/internal/pycore_pylifecycle.h

@@ -21,6 +21,9 @@ extern int _Py_SetFileSystemEncoding(
     const char *errors);
 extern void _Py_ClearFileSystemEncoding(void);
 extern _PyInitError _PyUnicode_InitEncodings(PyInterpreterState *interp);
+#ifdef MS_WINDOWS
+extern int _PyUnicode_EnableLegacyWindowsFSEncoding(void);
+#endif
 
 PyAPI_FUNC(void) _Py_ClearStandardStreamEncoding(void);
 

Include/internal/pycore_pystate.h

@@ -56,7 +56,14 @@ struct _is {
     PyObject *codec_search_cache;
     PyObject *codec_error_registry;
     int codecs_initialized;
-    int fscodec_initialized;
+
+    /* fs_codec.encoding is initialized to NULL.
+       Later, it is set to a non-NULL string by _PyUnicode_InitEncodings(). */
+    struct {
+        char *encoding;   /* Filesystem encoding (encoded to UTF-8) */
+        char *errors;     /* Filesystem errors (encoded to UTF-8) */
+        _Py_error_handler error_handler;
+    } fs_codec;
 
     _PyCoreConfig core_config;
 #ifdef HAVE_DLOPEN

Objects/stringlib/codecs.h

@@ -260,6 +260,7 @@ Py_LOCAL_INLINE(PyObject *)
 STRINGLIB(utf8_encoder)(PyObject *unicode,
                         STRINGLIB_CHAR *data,
                         Py_ssize_t size,
+                        _Py_error_handler error_handler,
                         const char *errors)
 {
     Py_ssize_t i;                /* index into data of next input character */
@@ -268,7 +269,6 @@ STRINGLIB(utf8_encoder)(PyObject *unicode,
     PyObject *error_handler_obj = NULL;
     PyObject *exc = NULL;
     PyObject *rep = NULL;
-    _Py_error_handler error_handler = _Py_ERROR_UNKNOWN;
 #endif
 #if STRINGLIB_SIZEOF_CHAR == 1
     const Py_ssize_t max_char_size = 2;