bpo-36301: Cleanup preconfig.c and coreconfig.c (GH-12563)

* _PyCoreConfig_Write() now updates _PyRuntime.preconfig
* Remove _PyPreCmdline_Copy()
* _PyPreCmdline_Read() now accepts _PyPreConfig and _PyCoreConfig
  optional configurations.
* Rename _PyPreConfig_ReadFromArgv() to _PyPreConfig_Read(). Simplify
  the code.
* Calling _PyCoreConfig_Read() no longer adds the warning options
  twice: don't add a warning option if it's already in the list.
* Rename _PyCoreConfig_ReadFromArgv() to _PyCoreConfig_Read().
* Rename config_from_cmdline() to _PyCoreConfig_ReadFromArgv().
* Add more assertions on _PyCoreConfig in _PyCoreConfig_Read().
* Move some functions.
* Make some config functions private.

Include/cpython/pylifecycle.h

@@ -21,8 +21,8 @@ PyAPI_FUNC(_PyInitError) _Py_PreInitializeFromConfig(
     const _PyCoreConfig *coreconfig);
 
 PyAPI_FUNC(_PyInitError) _Py_InitializeCore(
-    PyInterpreterState **interp,
+    const _PyCoreConfig *config,
-    const _PyCoreConfig *);
+    PyInterpreterState **interp);
 PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
 
 

Include/internal/pycore_coreconfig.h

@@ -14,8 +14,8 @@ extern "C" {
 typedef struct {
     _PyWstrList argv;
     _PyWstrList xoptions;     /* "-X value" option */
-    int use_environment;      /* -E option */
     int isolated;             /* -I option */
+    int use_environment;      /* -E option */
     int dev_mode;             /* -X dev and PYTHONDEVMODE */
 } _PyPreCmdline;
 
@@ -27,23 +27,14 @@ typedef struct {
 /* Note: _PyPreCmdline_INIT sets other fields to 0/NULL */
 
 PyAPI_FUNC(void) _PyPreCmdline_Clear(_PyPreCmdline *cmdline);
-PyAPI_FUNC(int) _PyPreCmdline_Copy(_PyPreCmdline *cmdline,
-    const _PyPreCmdline *cmdline2);
 PyAPI_FUNC(_PyInitError) _PyPreCmdline_SetArgv(_PyPreCmdline *cmdline,
     const _PyArgv *args);
-PyAPI_FUNC(void) _PyPreCmdline_GetPreConfig(
+PyAPI_FUNC(int) _PyPreCmdline_SetCoreConfig(
-    _PyPreCmdline *cmdline,
-    const _PyPreConfig *config);
-PyAPI_FUNC(void) _PyPreCmdline_SetPreConfig(
-    const _PyPreCmdline *cmdline,
-    _PyPreConfig *config);
-PyAPI_FUNC(void) _PyPreCmdline_GetCoreConfig(
-    _PyPreCmdline *cmdline,
-    const _PyCoreConfig *config);
-PyAPI_FUNC(void) _PyPreCmdline_SetCoreConfig(
     const _PyPreCmdline *cmdline,
     _PyCoreConfig *config);
-PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline);
+PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline,
+    const _PyPreConfig *preconfig,
+    const _PyCoreConfig *coreconfig);
 
 
 /* --- _PyWstrList ------------------------------------------------ */
@@ -57,6 +48,8 @@ PyAPI_FUNC(int) _PyWstrList_Copy(_PyWstrList *list,
 PyAPI_FUNC(int) _PyWstrList_Append(_PyWstrList *list,
     const wchar_t *item);
 PyAPI_FUNC(PyObject*) _PyWstrList_AsList(const _PyWstrList *list);
+PyAPI_FUNC(int) _PyWstrList_Extend(_PyWstrList *list,
+    const _PyWstrList *list2);
 
 
 /* --- _PyArgv ---------------------------------------------------- */
@@ -70,7 +63,7 @@ PyAPI_FUNC(_PyInitError) _PyArgv_AsWstrList(const _PyArgv *args,
 PyAPI_FUNC(void) _Py_ClearArgcArgv(void);
 
 
-/* --- _PyPreConfig ----------------------------------------------- */
+/* --- Helper functions ------------------------------------------- */
 
 PyAPI_FUNC(int) _Py_str_to_int(
     const char *str,
@@ -81,22 +74,20 @@ PyAPI_FUNC(const wchar_t*) _Py_get_xoption(
 PyAPI_FUNC(const char*) _Py_GetEnv(
     int use_environment,
     const char *name);
-
-PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config);
-PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
-    const _PyPreConfig *config2);
-PyAPI_FUNC(void) _PyPreConfig_GetGlobalConfig(_PyPreConfig *config);
-PyAPI_FUNC(void) _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config);
 PyAPI_FUNC(void) _Py_get_env_flag(
     int use_environment,
     int *flag,
     const char *name);
+
+/* --- _PyPreConfig ----------------------------------------------- */
+
+PyAPI_FUNC(void) _PyPreConfig_Clear(_PyPreConfig *config);
+PyAPI_FUNC(int) _PyPreConfig_Copy(_PyPreConfig *config,
+    const _PyPreConfig *config2);
+PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config,
     const _PyArgv *args,
     const _PyCoreConfig *coreconfig);
-PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config);
-PyAPI_FUNC(_PyInitError) _PyPreConfig_ReadFromArgv(_PyPreConfig *config,
-    const _PyArgv *args);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(_PyPreConfig *config);
 
 
@@ -109,10 +100,7 @@ PyAPI_FUNC(int) _PyCoreConfig_Copy(
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPathConfig(_PyCoreConfig *config);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetPathConfig(
     const _PyCoreConfig *config);
-PyAPI_FUNC(void) _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config,
-PyAPI_FUNC(void) _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config,
     const _PyArgv *args);
 PyAPI_FUNC(void) _PyCoreConfig_Write(const _PyCoreConfig *config);
 

Lib/test/test_embed.py

@@ -302,7 +302,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'pycache_prefix': None,
         'program_name': './_testembed',
         'argv': [""],
-        'program': None,
+        'program': '',
 
         'xoptions': [],
         'warnoptions': [],
@@ -537,6 +537,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'program_name': './globalvar',
             'site_import': 0,
             'bytes_warning': 1,
+            'warnoptions': ['default::BytesWarning'],
             'inspect': 1,
             'interactive': 1,
             'optimization_level': 2,
@@ -579,7 +580,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'argv': ['-c', 'pass'],
             'program': 'conf_program',
             'xoptions': ['core_xoption1=3', 'core_xoption2=', 'core_xoption3'],
-            'warnoptions': ['default', 'error::ResourceWarning'],
+            'warnoptions': ['error::ResourceWarning', 'default::BytesWarning'],
 
             'site_import': 0,
             'bytes_warning': 1,
@@ -629,14 +630,16 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         preconfig = dict(self.INIT_ENV_PRECONFIG,
                       allocator='debug')
         config = dict(self.INIT_ENV_CONFIG,
-                      dev_mode=1)
+                      dev_mode=1,
+                      warnoptions=['default'])
         self.check_config("init_env_dev_mode", config, preconfig)
 
     def test_init_env_dev_mode_alloc(self):
         preconfig = dict(self.INIT_ENV_PRECONFIG,
                          allocator='malloc')
         config = dict(self.INIT_ENV_CONFIG,
-                      dev_mode=1)
+                      dev_mode=1,
+                      warnoptions=['default'])
         self.check_config("init_env_dev_mode_alloc", config, preconfig)
 
     def test_init_dev_mode(self):
@@ -646,14 +649,12 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         config = {
             'faulthandler': 1,
             'dev_mode': 1,
+            'warnoptions': ['default'],
         }
         self.check_config("init_dev_mode", config, preconfig)
 
     def test_init_isolated(self):
-        preconfig = {
+        preconfig = {}
-            'isolated': 0,
-            'use_environment': 1,
-        }
         config = {
             'isolated': 1,
             'use_environment': 0,

Modules/main.c

@@ -289,7 +289,7 @@ pymain_init_preconfig(const _PyArgv *args)
 
     _PyPreConfig config = _PyPreConfig_INIT;
 
-    err = _PyPreConfig_ReadFromArgv(&config, args);
+    err = _PyPreConfig_Read(&config, args, NULL);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
@@ -306,12 +306,12 @@ static _PyInitError
 pymain_init_coreconfig(_PyCoreConfig *config, const _PyArgv *args,
                        PyInterpreterState **interp_p)
 {
-    _PyInitError err = _PyCoreConfig_ReadFromArgv(config, args);
+    _PyInitError err = _PyCoreConfig_Read(config, args);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
-    return _Py_InitializeCore(interp_p, config);
+    return _Py_InitializeCore(config, interp_p);
 }
 
 
@@ -359,22 +359,18 @@ pymain_init(const _PyArgv *args, PyInterpreterState **interp_p)
     }
 
     _PyCoreConfig config = _PyCoreConfig_INIT;
-
     err = pymain_init_coreconfig(&config, args, interp_p);
+    _PyCoreConfig_Clear(&config);
     if (_Py_INIT_FAILED(err)) {
-        goto done;
+        return err;
     }
 
     err = pymain_init_python_main(*interp_p);
     if (_Py_INIT_FAILED(err)) {
-        goto done;
+        return err;
     }
 
-    err = _Py_INIT_OK();
+    return _Py_INIT_OK();
-
-done:
-    _PyCoreConfig_Clear(&config);
-    return err;
 }
 
 

Programs/_testembed.c

@@ -466,8 +466,7 @@ static int test_init_from_config(void)
     config.xoptions.length = Py_ARRAY_LENGTH(xoptions);
     config.xoptions.items = xoptions;
 
-    static wchar_t* warnoptions[2] = {
+    static wchar_t* warnoptions[1] = {
-        L"default",
         L"error::ResourceWarning",
     };
     config.warnoptions.length = Py_ARRAY_LENGTH(warnoptions);

Python/coreconfig.c

@@ -296,7 +296,7 @@ _PyWstrList_Append(_PyWstrList *list, const wchar_t *item)
 }
 
 
-static int
+int
 _PyWstrList_Extend(_PyWstrList *list, const _PyWstrList *list2)
 {
     for (Py_ssize_t i = 0; i < list2->length; i++) {
@@ -308,6 +308,18 @@ _PyWstrList_Extend(_PyWstrList *list, const _PyWstrList *list2)
 }
 
 
+static int
+_PyWstrList_Find(_PyWstrList *list, const wchar_t *item)
+{
+    for (Py_ssize_t i = 0; i < list->length; i++) {
+        if (wcscmp(list->items[i], item) == 0) {
+            return 1;
+        }
+    }
+    return 0;
+}
+
+
 PyObject*
 _PyWstrList_AsList(const _PyWstrList *list)
 {
@@ -607,6 +619,120 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
 }
 
 
+static PyObject *
+_PyCoreConfig_AsDict(const _PyCoreConfig *config)
+{
+    PyObject *dict;
+
+    dict = PyDict_New();
+    if (dict == NULL) {
+        return NULL;
+    }
+
+#define SET_ITEM(KEY, EXPR) \
+        do { \
+            PyObject *obj = (EXPR); \
+            if (obj == NULL) { \
+                goto fail; \
+            } \
+            int res = PyDict_SetItemString(dict, (KEY), obj); \
+            Py_DECREF(obj); \
+            if (res < 0) { \
+                goto fail; \
+            } \
+        } while (0)
+#define FROM_STRING(STR) \
+    ((STR != NULL) ? \
+        PyUnicode_FromString(STR) \
+        : (Py_INCREF(Py_None), Py_None))
+#define SET_ITEM_INT(ATTR) \
+    SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
+#define SET_ITEM_UINT(ATTR) \
+    SET_ITEM(#ATTR, PyLong_FromUnsignedLong(config->ATTR))
+#define SET_ITEM_STR(ATTR) \
+    SET_ITEM(#ATTR, FROM_STRING(config->ATTR))
+#define FROM_WSTRING(STR) \
+    ((STR != NULL) ? \
+        PyUnicode_FromWideChar(STR, -1) \
+        : (Py_INCREF(Py_None), Py_None))
+#define SET_ITEM_WSTR(ATTR) \
+    SET_ITEM(#ATTR, FROM_WSTRING(config->ATTR))
+#define SET_ITEM_WSTRLIST(LIST) \
+    SET_ITEM(#LIST, _PyWstrList_AsList(&config->LIST))
+
+    SET_ITEM_INT(isolated);
+    SET_ITEM_INT(use_environment);
+    SET_ITEM_INT(dev_mode);
+    SET_ITEM_INT(install_signal_handlers);
+    SET_ITEM_INT(use_hash_seed);
+    SET_ITEM_UINT(hash_seed);
+    SET_ITEM_INT(faulthandler);
+    SET_ITEM_INT(tracemalloc);
+    SET_ITEM_INT(import_time);
+    SET_ITEM_INT(show_ref_count);
+    SET_ITEM_INT(show_alloc_count);
+    SET_ITEM_INT(dump_refs);
+    SET_ITEM_INT(malloc_stats);
+    SET_ITEM_STR(filesystem_encoding);
+    SET_ITEM_STR(filesystem_errors);
+    SET_ITEM_WSTR(pycache_prefix);
+    SET_ITEM_WSTR(program_name);
+    SET_ITEM_WSTRLIST(argv);
+    SET_ITEM_WSTR(program);
+    SET_ITEM_WSTRLIST(xoptions);
+    SET_ITEM_WSTRLIST(warnoptions);
+    SET_ITEM_WSTR(module_search_path_env);
+    SET_ITEM_WSTR(home);
+    SET_ITEM_WSTRLIST(module_search_paths);
+    SET_ITEM_WSTR(executable);
+    SET_ITEM_WSTR(prefix);
+    SET_ITEM_WSTR(base_prefix);
+    SET_ITEM_WSTR(exec_prefix);
+    SET_ITEM_WSTR(base_exec_prefix);
+#ifdef MS_WINDOWS
+    SET_ITEM_WSTR(dll_path);
+#endif
+    SET_ITEM_INT(site_import);
+    SET_ITEM_INT(bytes_warning);
+    SET_ITEM_INT(inspect);
+    SET_ITEM_INT(interactive);
+    SET_ITEM_INT(optimization_level);
+    SET_ITEM_INT(parser_debug);
+    SET_ITEM_INT(write_bytecode);
+    SET_ITEM_INT(verbose);
+    SET_ITEM_INT(quiet);
+    SET_ITEM_INT(user_site_directory);
+    SET_ITEM_INT(buffered_stdio);
+    SET_ITEM_STR(stdio_encoding);
+    SET_ITEM_STR(stdio_errors);
+#ifdef MS_WINDOWS
+    SET_ITEM_INT(legacy_windows_stdio);
+#endif
+    SET_ITEM_INT(skip_source_first_line);
+    SET_ITEM_WSTR(run_command);
+    SET_ITEM_WSTR(run_module);
+    SET_ITEM_WSTR(run_filename);
+    SET_ITEM_INT(_install_importlib);
+    SET_ITEM_STR(_check_hash_pycs_mode);
+    SET_ITEM_INT(_frozen);
+
+    return dict;
+
+fail:
+    Py_DECREF(dict);
+    return NULL;
+
+#undef FROM_STRING
+#undef FROM_WSTRING
+#undef SET_ITEM
+#undef SET_ITEM_INT
+#undef SET_ITEM_UINT
+#undef SET_ITEM_STR
+#undef SET_ITEM_WSTR
+#undef SET_ITEM_WSTRLIST
+}
+
+
 static const char*
 _PyCoreConfig_GetEnv(const _PyCoreConfig *config, const char *name)
 {
@@ -614,6 +740,9 @@ _PyCoreConfig_GetEnv(const _PyCoreConfig *config, const char *name)
 }
 
 
+/* Get a copy of the environment variable as wchar_t*.
+   Return 0 on success, but *dest can be NULL.
+   Return -1 on memory allocation failure. Return -2 on decoding error. */
 static int
 _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config,
                         wchar_t **dest,
@@ -662,7 +791,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config,
 }
 
 
-void
+static void
 _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config)
 {
 #define COPY_FLAG(ATTR, VALUE) \
@@ -699,7 +828,7 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config)
 
 
 /* Set Py_xxx global configuration variables from 'config' configuration. */
-void
+static void
 _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config)
 {
 #define COPY_FLAG(ATTR, VAR) \
@@ -848,10 +977,10 @@ config_get_xoption(const _PyCoreConfig *config, wchar_t *name)
 static _PyInitError
 config_init_home(_PyCoreConfig *config)
 {
-    wchar_t *home;
+    assert(config->home == NULL);
 
     /* If Py_SetPythonHome() was called, use its value */
-    home = _Py_path_config.home;
+    wchar_t *home = _Py_path_config.home;
     if (home) {
         config->home = _PyMem_RawWcsdup(home);
         if (config->home == NULL) {
@@ -1098,7 +1227,7 @@ config_read_complex_options(_PyCoreConfig *config)
 
 
 static const char *
-get_stdio_errors(const _PyCoreConfig *config)
+config_get_stdio_errors(const _PyCoreConfig *config)
 {
 #ifndef MS_WINDOWS
     const char *loc = setlocale(LC_CTYPE, NULL);
@@ -1125,7 +1254,7 @@ get_stdio_errors(const _PyCoreConfig *config)
 
 
 static _PyInitError
-get_locale_encoding(char **locale_encoding)
+config_get_locale_encoding(char **locale_encoding)
 {
 #ifdef MS_WINDOWS
     char encoding[20];
@@ -1236,13 +1365,13 @@ config_init_stdio_encoding(_PyCoreConfig *config,
 
     /* Choose the default error handler based on the current locale. */
     if (config->stdio_encoding == NULL) {
-        _PyInitError err = get_locale_encoding(&config->stdio_encoding);
+        _PyInitError err = config_get_locale_encoding(&config->stdio_encoding);
         if (_Py_INIT_FAILED(err)) {
             return err;
         }
     }
     if (config->stdio_errors == NULL) {
-        const char *errors = get_stdio_errors(config);
+        const char *errors = config_get_stdio_errors(config);
         config->stdio_errors = _PyMem_RawStrdup(errors);
         if (config->stdio_errors == NULL) {
             return _Py_INIT_NO_MEMORY();
@@ -1306,7 +1435,7 @@ config_init_fs_encoding(_PyCoreConfig *config, const _PyPreConfig *preconfig)
 #if defined(__APPLE__) || defined(__ANDROID__)
             config->filesystem_encoding = _PyMem_RawStrdup("utf-8");
 #else
-            _PyInitError err = get_locale_encoding(&config->filesystem_encoding);
+            _PyInitError err = config_get_locale_encoding(&config->filesystem_encoding);
             if (_Py_INIT_FAILED(err)) {
                 return err;
             }
@@ -1330,38 +1459,22 @@ config_init_fs_encoding(_PyCoreConfig *config, const _PyPreConfig *preconfig)
 }
 
 
-/* Read the configuration into _PyCoreConfig from:
-
-   * Environment variables
-   * Py_xxx global configuration variables
-
-   See _PyCoreConfig_ReadFromArgv() to parse also command line arguments. */
 static _PyInitError
-config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline)
+config_read(_PyCoreConfig *config, _PyPreCmdline *cmdline)
 {
     _PyInitError err;
 
     const _PyPreConfig *preconfig = &_PyRuntime.preconfig;
-    _PyPreCmdline_GetPreConfig(cmdline, preconfig);
+    err = _PyPreCmdline_Read(cmdline, preconfig, config);
-    _PyPreCmdline_GetCoreConfig(cmdline, config);
-
-    err = _PyPreCmdline_Read(cmdline);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
-    _PyPreCmdline_SetCoreConfig(cmdline, config);
+    if (_PyPreCmdline_SetCoreConfig(cmdline, config) < 0) {
-
-    if (_PyWstrList_Extend(&config->xoptions, &cmdline->xoptions) < 0) {
         return _Py_INIT_NO_MEMORY();
     }
 
-    _PyCoreConfig_GetGlobalConfig(config);
-
-    assert(config->use_environment >= 0);
-
     if (config->isolated > 0) {
-        config->use_environment = 0;
         config->user_site_directory = 0;
     }
 
@@ -1419,16 +1532,16 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline)
             config->faulthandler = 1;
         }
     }
-    if (config->use_hash_seed < 0) {
-        config->use_hash_seed = 0;
-        config->hash_seed = 0;
-    }
     if (config->faulthandler < 0) {
         config->faulthandler = 0;
     }
     if (config->tracemalloc < 0) {
         config->tracemalloc = 0;
     }
+    if (config->use_hash_seed < 0) {
+        config->use_hash_seed = 0;
+        config->hash_seed = 0;
+    }
 
     if (config->filesystem_encoding == NULL || config->filesystem_errors == NULL) {
         err = config_init_fs_encoding(config, preconfig);
@@ -1449,53 +1562,10 @@ config_read_impl(_PyCoreConfig *config, _PyPreCmdline *cmdline)
         }
     }
 
-    assert(config->use_environment >= 0);
-    assert(config->filesystem_encoding != NULL);
-    assert(config->filesystem_errors != NULL);
-    assert(config->stdio_encoding != NULL);
-    assert(config->stdio_errors != NULL);
-    assert(config->_check_hash_pycs_mode != NULL);
-    assert(_PyWstrList_CheckConsistency(&config->argv));
-
     return _Py_INIT_OK();
 }
 
 
-static _PyInitError
-config_read(_PyCoreConfig *config, const _PyPreCmdline *src_cmdline)
-{
-    _PyInitError err;
-
-    err = _Py_PreInitializeFromConfig(config);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    _PyPreCmdline cmdline = _PyPreCmdline_INIT;
-
-    if (src_cmdline) {
-        if (_PyPreCmdline_Copy(&cmdline, src_cmdline) < 0) {
-            err = _Py_INIT_NO_MEMORY();
-            goto done;
-        }
-    }
-
-    err = config_read_impl(config, &cmdline);
-
-done:
-    _PyPreCmdline_Clear(&cmdline);
-    return err;
-
-}
-
-
-_PyInitError
-_PyCoreConfig_Read(_PyCoreConfig *config)
-{
-    return config_read(config, NULL);
-}
-
-
 static void
 config_init_stdio(const _PyCoreConfig *config)
 {
@@ -1542,139 +1612,30 @@ _PyCoreConfig_Write(const _PyCoreConfig *config)
 {
     _PyCoreConfig_SetGlobalConfig(config);
     config_init_stdio(config);
-}
 
-
+    /* Write the new pre-configuration into _PyRuntime */
-static PyObject *
+    _PyPreConfig *preconfig = &_PyRuntime.preconfig;
-_PyCoreConfig_AsDict(const _PyCoreConfig *config)
+    preconfig->isolated = config->isolated;
-{
+    preconfig->use_environment = config->use_environment;
-    PyObject *dict;
+    preconfig->dev_mode = config->dev_mode;
-
-    dict = PyDict_New();
-    if (dict == NULL) {
-        return NULL;
-    }
-
-#define SET_ITEM(KEY, EXPR) \
-        do { \
-            PyObject *obj = (EXPR); \
-            if (obj == NULL) { \
-                goto fail; \
-            } \
-            int res = PyDict_SetItemString(dict, (KEY), obj); \
-            Py_DECREF(obj); \
-            if (res < 0) { \
-                goto fail; \
-            } \
-        } while (0)
-#define FROM_STRING(STR) \
-    ((STR != NULL) ? \
-        PyUnicode_FromString(STR) \
-        : (Py_INCREF(Py_None), Py_None))
-#define SET_ITEM_INT(ATTR) \
-    SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
-#define SET_ITEM_UINT(ATTR) \
-    SET_ITEM(#ATTR, PyLong_FromUnsignedLong(config->ATTR))
-#define SET_ITEM_STR(ATTR) \
-    SET_ITEM(#ATTR, FROM_STRING(config->ATTR))
-#define FROM_WSTRING(STR) \
-    ((STR != NULL) ? \
-        PyUnicode_FromWideChar(STR, -1) \
-        : (Py_INCREF(Py_None), Py_None))
-#define SET_ITEM_WSTR(ATTR) \
-    SET_ITEM(#ATTR, FROM_WSTRING(config->ATTR))
-#define SET_ITEM_WSTRLIST(LIST) \
-    SET_ITEM(#LIST, _PyWstrList_AsList(&config->LIST))
-
-    SET_ITEM_INT(isolated);
-    SET_ITEM_INT(use_environment);
-    SET_ITEM_INT(dev_mode);
-    SET_ITEM_INT(install_signal_handlers);
-    SET_ITEM_INT(use_hash_seed);
-    SET_ITEM_UINT(hash_seed);
-    SET_ITEM_INT(faulthandler);
-    SET_ITEM_INT(tracemalloc);
-    SET_ITEM_INT(import_time);
-    SET_ITEM_INT(show_ref_count);
-    SET_ITEM_INT(show_alloc_count);
-    SET_ITEM_INT(dump_refs);
-    SET_ITEM_INT(malloc_stats);
-    SET_ITEM_STR(filesystem_encoding);
-    SET_ITEM_STR(filesystem_errors);
-    SET_ITEM_WSTR(pycache_prefix);
-    SET_ITEM_WSTR(program_name);
-    SET_ITEM_WSTRLIST(argv);
-    SET_ITEM_WSTR(program);
-    SET_ITEM_WSTRLIST(xoptions);
-    SET_ITEM_WSTRLIST(warnoptions);
-    SET_ITEM_WSTR(module_search_path_env);
-    SET_ITEM_WSTR(home);
-    SET_ITEM_WSTRLIST(module_search_paths);
-    SET_ITEM_WSTR(executable);
-    SET_ITEM_WSTR(prefix);
-    SET_ITEM_WSTR(base_prefix);
-    SET_ITEM_WSTR(exec_prefix);
-    SET_ITEM_WSTR(base_exec_prefix);
-#ifdef MS_WINDOWS
-    SET_ITEM_WSTR(dll_path);
-#endif
-    SET_ITEM_INT(site_import);
-    SET_ITEM_INT(bytes_warning);
-    SET_ITEM_INT(inspect);
-    SET_ITEM_INT(interactive);
-    SET_ITEM_INT(optimization_level);
-    SET_ITEM_INT(parser_debug);
-    SET_ITEM_INT(write_bytecode);
-    SET_ITEM_INT(verbose);
-    SET_ITEM_INT(quiet);
-    SET_ITEM_INT(user_site_directory);
-    SET_ITEM_INT(buffered_stdio);
-    SET_ITEM_STR(stdio_encoding);
-    SET_ITEM_STR(stdio_errors);
-#ifdef MS_WINDOWS
-    SET_ITEM_INT(legacy_windows_stdio);
-#endif
-    SET_ITEM_INT(skip_source_first_line);
-    SET_ITEM_WSTR(run_command);
-    SET_ITEM_WSTR(run_module);
-    SET_ITEM_WSTR(run_filename);
-    SET_ITEM_INT(_install_importlib);
-    SET_ITEM_STR(_check_hash_pycs_mode);
-    SET_ITEM_INT(_frozen);
-
-    return dict;
-
-fail:
-    Py_DECREF(dict);
-    return NULL;
-
-#undef FROM_STRING
-#undef FROM_WSTRING
-#undef SET_ITEM
-#undef SET_ITEM_INT
-#undef SET_ITEM_UINT
-#undef SET_ITEM_STR
-#undef SET_ITEM_WSTR
-#undef SET_ITEM_WSTRLIST
 }
 
 
 /* --- _PyCmdline ------------------------------------------------- */
 
 typedef struct {
-    _PyPreCmdline precmdline;
+    _PyWstrList cmdline_warnoptions;     /* Command line -W options */
-    _PyWstrList warnoptions;     /* Command line -W options */
     _PyWstrList env_warnoptions; /* PYTHONWARNINGS environment variables */
     int print_help;              /* -h, -? options */
     int print_version;           /* -V option */
+    int need_usage;
 } _PyCmdline;
 
 
 static void
 cmdline_clear(_PyCmdline *cmdline)
 {
-    _PyPreCmdline_Clear(&cmdline->precmdline);
+    _PyWstrList_Clear(&cmdline->cmdline_warnoptions);
-    _PyWstrList_Clear(&cmdline->warnoptions);
     _PyWstrList_Clear(&cmdline->env_warnoptions);
 }
 
@@ -1684,9 +1645,9 @@ cmdline_clear(_PyCmdline *cmdline)
 /* Parse the command line arguments */
 static _PyInitError
 config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
-                     int *need_usage)
+                     _PyPreCmdline *precmdline)
 {
-    const _PyWstrList *argv = &cmdline->precmdline.argv;
+    const _PyWstrList *argv = &precmdline->argv;
 
     _PyOS_ResetGetOpt();
     do {
@@ -1740,7 +1701,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
             } else {
                 fprintf(stderr, "--check-hash-based-pycs must be one of "
                         "'default', 'always', or 'never'\n");
-                *need_usage = 1;
+                cmdline->need_usage = 1;
                 return _Py_INIT_OK();
             }
             break;
@@ -1761,7 +1722,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
         case 'E':
         case 'I':
         case 'X':
-            /* option handled by _PyPreConfig_ReadFromArgv() */
+            /* option handled by _PyPreCmdline_Read() */
             break;
 
         /* case 'J': reserved for Jython */
@@ -1808,7 +1769,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
             break;
 
         case 'W':
-            if (_PyWstrList_Append(&cmdline->warnoptions, _PyOS_optarg) < 0) {
+            if (_PyWstrList_Append(&cmdline->cmdline_warnoptions, _PyOS_optarg) < 0) {
                 return _Py_INIT_NO_MEMORY();
             }
             break;
@@ -1825,7 +1786,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
 
         default:
             /* unknown argument: parsing failed */
-            *need_usage = 1;
+            cmdline->need_usage = 1;
             return _Py_INIT_OK();
         }
     } while (1);
@@ -1891,9 +1852,9 @@ cmdline_init_env_warnoptions(_PyCmdline *cmdline, const _PyCoreConfig *config)
 
 
 static _PyInitError
-config_init_program(_PyCoreConfig *config, const _PyCmdline *cmdline)
+config_init_program(_PyCoreConfig *config, const _PyPreCmdline *cmdline)
 {
-    const _PyWstrList *argv = &cmdline->precmdline.argv;
+    const _PyWstrList *argv = &cmdline->argv;
     wchar_t *program;
     if (argv->length >= 1) {
         program = argv->items[0];
@@ -1910,11 +1871,23 @@ config_init_program(_PyCoreConfig *config, const _PyCmdline *cmdline)
 }
 
 
+static int
+config_add_warnoption(_PyCoreConfig *config, const wchar_t *option)
+{
+    if (_PyWstrList_Find(&config->warnoptions, option)) {
+        /* Already present: do nothing */
+        return 0;
+    }
+    if (_PyWstrList_Append(&config->warnoptions, option)) {
+        return -1;
+    }
+    return 0;
+}
+
+
 static _PyInitError
 config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
 {
-    assert(config->warnoptions.length == 0);
-
     /* The priority order for warnings configuration is (highest precedence
      * first):
      *
@@ -1931,18 +1904,27 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
      */
 
     if (config->dev_mode) {
-        if (_PyWstrList_Append(&config->warnoptions, L"default")) {
+        if (config_add_warnoption(config, L"default") < 0) {
             return _Py_INIT_NO_MEMORY();
         }
     }
 
-    if (_PyWstrList_Extend(&config->warnoptions, &cmdline->env_warnoptions) < 0) {
+    Py_ssize_t i;
+    const _PyWstrList *options;
+
+    options = &cmdline->env_warnoptions;
+    for (i = 0; i < options->length; i++) {
+        if (config_add_warnoption(config, options->items[i]) < 0) {
             return _Py_INIT_NO_MEMORY();
         }
+    }
 
-    if (_PyWstrList_Extend(&config->warnoptions, &cmdline->warnoptions) < 0) {
+    options = &cmdline->cmdline_warnoptions;
+    for (i = 0; i < options->length; i++) {
+        if (config_add_warnoption(config, options->items[i]) < 0) {
             return _Py_INIT_NO_MEMORY();
         }
+    }
 
     /* If the bytes_warning_flag isn't set, bytesobject.c and bytearrayobject.c
      * don't even try to emit a warning, so we skip setting the filter in that
@@ -1956,7 +1938,7 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
         else {
             filter = L"default::BytesWarning";
         }
-        if (_PyWstrList_Append(&config->warnoptions, filter)) {
+        if (config_add_warnoption(config, filter) < 0) {
             return _Py_INIT_NO_MEMORY();
         }
     }
@@ -1965,9 +1947,9 @@ config_init_warnoptions(_PyCoreConfig *config, const _PyCmdline *cmdline)
 
 
 static _PyInitError
-config_init_argv(_PyCoreConfig *config, const _PyCmdline *cmdline)
+config_init_argv(_PyCoreConfig *config, const _PyPreCmdline *cmdline)
 {
-    const _PyWstrList *cmdline_argv = &cmdline->precmdline.argv;
+    const _PyWstrList *cmdline_argv = &cmdline->argv;
     _PyWstrList config_argv = _PyWstrList_INIT;
 
     /* Copy argv to be able to modify it (to force -c/-m) */
@@ -2032,87 +2014,13 @@ config_usage(int error, const wchar_t* program)
 }
 
 
-/* Parse command line options and environment variables. */
-static _PyInitError
-config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline)
-{
-    int need_usage = 0;
-    _PyInitError err;
-
-    _PyCoreConfig_GetGlobalConfig(config);
-
-    if (config->program == NULL) {
-        err = config_init_program(config, cmdline);
-        if (_Py_INIT_FAILED(err)) {
-            return err;
-        }
-    }
-
-    err = _PyPreCmdline_Read(&cmdline->precmdline);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    _PyPreCmdline_SetPreConfig(&cmdline->precmdline, &_PyRuntime.preconfig);
-
-    err = config_parse_cmdline(config, cmdline, &need_usage);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    if (need_usage) {
-        config_usage(1, config->program);
-        return _Py_INIT_EXIT(2);
-    }
-
-    if (cmdline->print_help) {
-        config_usage(0, config->program);
-        return _Py_INIT_EXIT(0);
-    }
-
-    if (cmdline->print_version) {
-        printf("Python %s\n",
-               (cmdline->print_version >= 2) ? Py_GetVersion() : PY_VERSION);
-        return _Py_INIT_EXIT(0);
-    }
-
-    err = config_init_argv(config, cmdline);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    err = config_read(config, &cmdline->precmdline);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    if (config->use_environment) {
-        err = cmdline_init_env_warnoptions(cmdline, config);
-        if (_Py_INIT_FAILED(err)) {
-            return err;
-        }
-    }
-
-    err = config_init_warnoptions(config, cmdline);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    const _PyWstrList *argv = &cmdline->precmdline.argv;
-    if (_Py_SetArgcArgv(argv->length, argv->items) < 0) {
-        return _Py_INIT_NO_MEMORY();
-    }
-    return _Py_INIT_OK();
-}
-
-
 /* Read the configuration into _PyCoreConfig from:
 
    * Command line arguments
    * Environment variables
    * Py_xxx global configuration variables */
 _PyInitError
-_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args)
+_PyCoreConfig_Read(_PyCoreConfig *config, const _PyArgv *args)
 {
     _PyInitError err;
 
@@ -2121,21 +2029,137 @@ _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args)
         return err;
     }
 
-    _PyCmdline cmdline = {.precmdline = _PyPreCmdline_INIT};
+    _PyCoreConfig_GetGlobalConfig(config);
 
-    err = _PyPreCmdline_SetArgv(&cmdline.precmdline, args);
+    _PyPreCmdline precmdline = _PyPreCmdline_INIT;
+    if (args) {
+        err = _PyPreCmdline_SetArgv(&precmdline, args);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+    }
+
+    if (config->program == NULL) {
+        err = config_init_program(config, &precmdline);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+    }
+
+    const _PyPreConfig *preconfig = &_PyRuntime.preconfig;
+    err = _PyPreCmdline_Read(&precmdline, preconfig, config);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
-    err = config_from_cmdline(config, &cmdline);
+    _PyCmdline cmdline;
+    memset(&cmdline, 0, sizeof(cmdline));
+
+    if (args) {
+        err = config_parse_cmdline(config, &cmdline, &precmdline);
         if (_Py_INIT_FAILED(err)) {
             goto done;
         }
+
+        if (cmdline.need_usage) {
+            config_usage(1, config->program);
+            err = _Py_INIT_EXIT(2);
+            goto done;
+        }
+
+        if (cmdline.print_help) {
+            config_usage(0, config->program);
+            err = _Py_INIT_EXIT(0);
+            goto done;
+        }
+
+        if (cmdline.print_version) {
+            printf("Python %s\n",
+                   (cmdline.print_version >= 2) ? Py_GetVersion() : PY_VERSION);
+            err = _Py_INIT_EXIT(0);
+            goto done;
+        }
+
+        err = config_init_argv(config, &precmdline);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+    }
+
+    err = config_read(config, &precmdline);
+    if (_Py_INIT_FAILED(err)) {
+        goto done;
+    }
+
+    if (config->use_environment) {
+        err = cmdline_init_env_warnoptions(&cmdline, config);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+    }
+
+    err = config_init_warnoptions(config, &cmdline);
+    if (_Py_INIT_FAILED(err)) {
+        goto done;
+    }
+
+    const _PyWstrList *argv = &precmdline.argv;
+    if (_Py_SetArgcArgv(argv->length, argv->items) < 0) {
+        err = _Py_INIT_NO_MEMORY();
+        goto done;
+    }
+
+    /* Check config consistency */
+    assert(config->isolated >= 0);
+    assert(config->use_environment >= 0);
+    assert(config->dev_mode >= 0);
+    assert(config->install_signal_handlers >= 0);
+    assert(config->use_hash_seed >= 0);
+    assert(config->faulthandler >= 0);
+    assert(config->tracemalloc >= 0);
+    assert(config->site_import >= 0);
+    assert(config->bytes_warning >= 0);
+    assert(config->inspect >= 0);
+    assert(config->interactive >= 0);
+    assert(config->optimization_level >= 0);
+    assert(config->parser_debug >= 0);
+    assert(config->write_bytecode >= 0);
+    assert(config->verbose >= 0);
+    assert(config->quiet >= 0);
+    assert(config->user_site_directory >= 0);
+    assert(config->buffered_stdio >= 0);
+    assert(config->program_name != NULL);
+    assert(config->program != NULL);
+    assert(_PyWstrList_CheckConsistency(&config->argv));
+    assert(_PyWstrList_CheckConsistency(&config->xoptions));
+    assert(_PyWstrList_CheckConsistency(&config->warnoptions));
+    assert(_PyWstrList_CheckConsistency(&config->module_search_paths));
+    if (config->_install_importlib) {
+        assert(config->executable != NULL);
+        assert(config->prefix != NULL);
+        assert(config->base_prefix != NULL);
+        assert(config->exec_prefix != NULL);
+        assert(config->base_exec_prefix != NULL);
+#ifdef MS_WINDOWS
+        assert(config->dll_path != NULL);
+#endif
+    }
+    assert(config->filesystem_encoding != NULL);
+    assert(config->filesystem_errors != NULL);
+    assert(config->stdio_encoding != NULL);
+    assert(config->stdio_errors != NULL);
+#ifdef MS_WINDOWS
+    assert(config->legacy_windows_stdio >= 0);
+#endif
+    assert(config->_check_hash_pycs_mode != NULL);
+    assert(config->_install_importlib >= 0);
+    assert(config->_frozen >= 0);
+
     err = _Py_INIT_OK();
 
 done:
     cmdline_clear(&cmdline);
+    _PyPreCmdline_Clear(&precmdline);
     return err;
 }
 

Python/pathconfig.c

@@ -394,7 +394,7 @@ pathconfig_global_init(void)
     _PyInitError err;
     _PyCoreConfig config = _PyCoreConfig_INIT;
 
-    err = _PyCoreConfig_Read(&config);
+    err = _PyCoreConfig_Read(&config, NULL);
     if (_Py_INIT_FAILED(err)) {
         goto error;
     }

Python/preconfig.c

@@ -63,6 +63,7 @@ _Py_SetFileSystemEncoding(const char *encoding, const char *errors)
 
 /* --- _PyArgv ---------------------------------------------------- */
 
+/* Decode bytes_argv using Py_DecodeLocale() */
 _PyInitError
 _PyArgv_AsWstrList(const _PyArgv *args, _PyWstrList *list)
 {
@@ -110,22 +111,6 @@ _PyPreCmdline_Clear(_PyPreCmdline *cmdline)
 }
 
 
-int
-_PyPreCmdline_Copy(_PyPreCmdline *cmdline, const _PyPreCmdline *cmdline2)
-{
-    _PyPreCmdline_Clear(cmdline);
-    if (_PyWstrList_Copy(&cmdline->argv, &cmdline2->argv) < 0) {
-        return -1;
-    }
-    if (_PyWstrList_Copy(&cmdline->xoptions, &cmdline2->xoptions) < 0) {
-        return -1;
-    }
-    cmdline->use_environment = cmdline2->use_environment;
-    cmdline->isolated = cmdline2->isolated;
-    return 0;
-}
-
-
 _PyInitError
 _PyPreCmdline_SetArgv(_PyPreCmdline *cmdline, const _PyArgv *args)
 {
@@ -133,7 +118,7 @@ _PyPreCmdline_SetArgv(_PyPreCmdline *cmdline, const _PyArgv *args)
 }
 
 
-void
+static void
 _PyPreCmdline_GetPreConfig(_PyPreCmdline *cmdline, const _PyPreConfig *config)
 {
 #define COPY_ATTR(ATTR) \
@@ -141,31 +126,29 @@ _PyPreCmdline_GetPreConfig(_PyPreCmdline *cmdline, const _PyPreConfig *config)
         cmdline->ATTR = config->ATTR; \
     }
 
-    COPY_ATTR(use_environment);
     COPY_ATTR(isolated);
+    COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
 
 #undef COPY_ATTR
 }
 
 
-void
+static void
 _PyPreCmdline_SetPreConfig(const _PyPreCmdline *cmdline, _PyPreConfig *config)
 {
 #define COPY_ATTR(ATTR) \
-    if (cmdline->ATTR != -1) { \
+    config->ATTR = cmdline->ATTR
-        config->ATTR = cmdline->ATTR; \
-    }
 
-    COPY_ATTR(use_environment);
     COPY_ATTR(isolated);
+    COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
 
 #undef COPY_ATTR
 }
 
 
-void
+static void
 _PyPreCmdline_GetCoreConfig(_PyPreCmdline *cmdline, const _PyCoreConfig *config)
 {
 #define COPY_ATTR(ATTR) \
@@ -173,30 +156,126 @@ _PyPreCmdline_GetCoreConfig(_PyPreCmdline *cmdline, const _PyCoreConfig *config)
         cmdline->ATTR = config->ATTR; \
     }
 
-    COPY_ATTR(use_environment);
     COPY_ATTR(isolated);
+    COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
 
 #undef COPY_ATTR
 }
 
 
-void
+int
 _PyPreCmdline_SetCoreConfig(const _PyPreCmdline *cmdline, _PyCoreConfig *config)
 {
 #define COPY_ATTR(ATTR) \
-    if (config->ATTR == -1 && cmdline->ATTR != -1) { \
+    config->ATTR = cmdline->ATTR
-        config->ATTR = cmdline->ATTR; \
+
+    if (_PyWstrList_Extend(&config->xoptions, &cmdline->xoptions) < 0) {
+        return -1;
     }
 
-    COPY_ATTR(use_environment);
     COPY_ATTR(isolated);
+    COPY_ATTR(use_environment);
     COPY_ATTR(dev_mode);
+    return 0;
 
 #undef COPY_ATTR
 }
 
 
+/* Parse the command line arguments */
+static _PyInitError
+precmdline_parse_cmdline(_PyPreCmdline *cmdline)
+{
+    _PyWstrList *argv = &cmdline->argv;
+
+    _PyOS_ResetGetOpt();
+    /* Don't log parsing errors into stderr here: _PyCoreConfig_Read()
+       is responsible for that */
+    _PyOS_opterr = 0;
+    do {
+        int longindex = -1;
+        int c = _PyOS_GetOpt(argv->length, argv->items, &longindex);
+
+        if (c == EOF || c == 'c' || c == 'm') {
+            break;
+        }
+
+        switch (c) {
+        case 'E':
+            cmdline->use_environment = 0;
+            break;
+
+        case 'I':
+            cmdline->isolated = 1;
+            break;
+
+        case 'X':
+        {
+            if (_PyWstrList_Append(&cmdline->xoptions, _PyOS_optarg) < 0) {
+                return _Py_INIT_NO_MEMORY();
+            }
+            break;
+        }
+
+        default:
+            /* ignore other argument:
+               handled by _PyCoreConfig_Read() */
+            break;
+        }
+    } while (1);
+
+    return _Py_INIT_OK();
+}
+
+
+_PyInitError
+_PyPreCmdline_Read(_PyPreCmdline *cmdline,
+                    const _PyPreConfig *preconfig,
+                    const _PyCoreConfig *coreconfig)
+{
+    if (preconfig) {
+        _PyPreCmdline_GetPreConfig(cmdline, preconfig);
+    }
+
+    if (coreconfig) {
+        _PyPreCmdline_GetCoreConfig(cmdline, coreconfig);
+    }
+
+    _PyInitError err = precmdline_parse_cmdline(cmdline);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    /* isolated, use_environment */
+    if (cmdline->isolated < 0) {
+        cmdline->isolated = 0;
+    }
+    if (cmdline->isolated > 0) {
+        cmdline->use_environment = 0;
+    }
+    if (cmdline->use_environment < 0) {
+        cmdline->use_environment = 0;
+    }
+
+    /* dev_mode */
+    if ((cmdline && _Py_get_xoption(&cmdline->xoptions, L"dev"))
+        || _Py_GetEnv(cmdline->use_environment, "PYTHONDEVMODE"))
+    {
+        cmdline->dev_mode = 1;
+    }
+    if (cmdline->dev_mode < 0) {
+        cmdline->dev_mode = 0;
+    }
+
+    assert(cmdline->use_environment >= 0);
+    assert(cmdline->isolated >= 0);
+    assert(cmdline->dev_mode >= 0);
+
+    return _Py_INIT_OK();
+}
+
+
 /* --- _PyPreConfig ----------------------------------------------- */
 
 void
@@ -240,7 +319,61 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
 }
 
 
-void
+PyObject*
+_PyPreConfig_AsDict(const _PyPreConfig *config)
+{
+    PyObject *dict;
+
+    dict = PyDict_New();
+    if (dict == NULL) {
+        return NULL;
+    }
+
+#define SET_ITEM(KEY, EXPR) \
+        do { \
+            PyObject *obj = (EXPR); \
+            if (obj == NULL) { \
+                goto fail; \
+            } \
+            int res = PyDict_SetItemString(dict, (KEY), obj); \
+            Py_DECREF(obj); \
+            if (res < 0) { \
+                goto fail; \
+            } \
+        } while (0)
+#define SET_ITEM_INT(ATTR) \
+    SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
+#define FROM_STRING(STR) \
+    ((STR != NULL) ? \
+        PyUnicode_FromString(STR) \
+        : (Py_INCREF(Py_None), Py_None))
+#define SET_ITEM_STR(ATTR) \
+    SET_ITEM(#ATTR, FROM_STRING(config->ATTR))
+
+    SET_ITEM_INT(isolated);
+    SET_ITEM_INT(use_environment);
+    SET_ITEM_INT(coerce_c_locale);
+    SET_ITEM_INT(coerce_c_locale_warn);
+    SET_ITEM_INT(utf8_mode);
+#ifdef MS_WINDOWS
+    SET_ITEM_INT(legacy_windows_fs_encoding);
+#endif
+    SET_ITEM_INT(dev_mode);
+    SET_ITEM_STR(allocator);
+    return dict;
+
+fail:
+    Py_DECREF(dict);
+    return NULL;
+
+#undef FROM_STRING
+#undef SET_ITEM
+#undef SET_ITEM_INT
+#undef SET_ITEM_STR
+}
+
+
+static void
 _PyPreConfig_GetGlobalConfig(_PyPreConfig *config)
 {
 #define COPY_FLAG(ATTR, VALUE) \
@@ -264,7 +397,7 @@ _PyPreConfig_GetGlobalConfig(_PyPreConfig *config)
 }
 
 
-void
+static void
 _PyPreConfig_SetGlobalConfig(const _PyPreConfig *config)
 {
 #define COPY_FLAG(ATTR, VAR) \
@@ -307,13 +440,6 @@ _Py_GetEnv(int use_environment, const char *name)
 }
 
 
-static const char*
-_PyPreConfig_GetEnv(const _PyPreConfig *config, const char *name)
-{
-    return _Py_GetEnv(config->use_environment, name);
-}
-
-
 int
 _Py_str_to_int(const char *str, int *result)
 {
@@ -374,6 +500,16 @@ _Py_get_xoption(const _PyWstrList *xoptions, const wchar_t *name)
 static _PyInitError
 preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline)
 {
+#ifdef MS_WINDOWS
+    if (config->legacy_windows_fs_encoding) {
+        config->utf8_mode = 0;
+    }
+#endif
+
+    if (config->utf8_mode >= 0) {
+        return _Py_INIT_OK();
+    }
+
     const wchar_t *xopt;
     if (cmdline) {
         xopt = _Py_get_xoption(&cmdline->xoptions, L"utf8");
@@ -401,7 +537,7 @@ preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline)
         return _Py_INIT_OK();
     }
 
-    const char *opt = _PyPreConfig_GetEnv(config, "PYTHONUTF8");
+    const char *opt = _Py_GetEnv(config->use_environment, "PYTHONUTF8");
     if (opt) {
         if (strcmp(opt, "1") == 0) {
             config->utf8_mode = 1;
@@ -416,56 +552,31 @@ preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline)
         return _Py_INIT_OK();
     }
 
+
+#ifndef MS_WINDOWS
+    if (config->utf8_mode < 0) {
+        /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */
+        const char *ctype_loc = setlocale(LC_CTYPE, NULL);
+        if (ctype_loc != NULL
+           && (strcmp(ctype_loc, "C") == 0
+               || strcmp(ctype_loc, "POSIX") == 0))
+        {
+            config->utf8_mode = 1;
+        }
+    }
+#endif
+
+    if (config->utf8_mode < 0) {
+        config->utf8_mode = 0;
+    }
     return _Py_INIT_OK();
 }
 
 
 static void
-preconfig_init_locale(_PyPreConfig *config)
+preconfig_init_coerce_c_locale(_PyPreConfig *config)
 {
-    /* The C locale enables the C locale coercion (PEP 538) */
+    const char *env = _Py_GetEnv(config->use_environment, "PYTHONCOERCECLOCALE");
-    if (_Py_LegacyLocaleDetected()) {
-        config->coerce_c_locale = 2;
-    }
-    else {
-        config->coerce_c_locale = 0;
-    }
-}
-
-
-static _PyInitError
-preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline)
-{
-    _PyInitError err;
-
-    err = _PyPreCmdline_Read(cmdline);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    _PyPreCmdline_SetPreConfig(cmdline, config);
-
-    _PyPreConfig_GetGlobalConfig(config);
-
-    /* isolated and use_environment */
-    if (config->isolated > 0) {
-        config->use_environment = 0;
-    }
-
-    /* Default values */
-    if (config->use_environment < 0) {
-        config->use_environment = 0;
-    }
-
-    /* legacy_windows_fs_encoding, utf8_mode, coerce_c_locale */
-    if (config->use_environment) {
-#ifdef MS_WINDOWS
-        _Py_get_env_flag(config->use_environment,
-                         &config->legacy_windows_fs_encoding,
-                         "PYTHONLEGACYWINDOWSFSENCODING");
-#endif
-
-        const char *env = _PyPreConfig_GetEnv(config, "PYTHONCOERCECLOCALE");
     if (env) {
         if (strcmp(env, "0") == 0) {
             if (config->coerce_c_locale < 0) {
@@ -481,68 +592,35 @@ preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline)
             }
         }
     }
+
+    /* Test if coerce_c_locale equals to -1 or equals to 1:
+       PYTHONCOERCECLOCALE=1 doesn't imply that the C locale is always coerced.
+       It is only coerced if if the LC_CTYPE locale is "C". */
+    if (config->coerce_c_locale == 0 || config->coerce_c_locale == 2) {
+        return;
     }
 
-#ifdef MS_WINDOWS
+    /* The C locale enables the C locale coercion (PEP 538) */
-    if (config->legacy_windows_fs_encoding) {
+    if (_Py_LegacyLocaleDetected()) {
-        config->utf8_mode = 0;
+        config->coerce_c_locale = 2;
     }
-#endif
+    else {
-
-    if (config->utf8_mode < 0) {
-        err = preconfig_init_utf8_mode(config, cmdline);
-        if (_Py_INIT_FAILED(err)) {
-            return err;
-        }
-    }
-
-    /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't
-       imply that the C locale is always coerced. It is only coerced if
-       if the LC_CTYPE locale is "C". */
-    if (config->coerce_c_locale != 0 && config->coerce_c_locale != 2) {
-        preconfig_init_locale(config);
-    }
-
-#ifndef MS_WINDOWS
-    if (config->utf8_mode < 0) {
-        /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */
-        const char *ctype_loc = setlocale(LC_CTYPE, NULL);
-        if (ctype_loc != NULL
-           && (strcmp(ctype_loc, "C") == 0
-               || strcmp(ctype_loc, "POSIX") == 0))
-        {
-            config->utf8_mode = 1;
-        }
-    }
-#endif
-
-    if (config->coerce_c_locale < 0) {
-        config->coerce_c_locale = 0;
-    }
-    if (config->utf8_mode < 0) {
-        config->utf8_mode = 0;
-    }
-    if (config->coerce_c_locale < 0) {
         config->coerce_c_locale = 0;
     }
 
-    /* dev_mode */
+    assert(config->coerce_c_locale >= 0);
-    if ((cmdline && _Py_get_xoption(&cmdline->xoptions, L"dev"))
+}
-        || _PyPreConfig_GetEnv(config, "PYTHONDEVMODE"))
-    {
-        config->dev_mode = 1;
-    }
-    if (config->dev_mode < 0) {
-        config->dev_mode = 0;
-    }
 
-    /* allocator */
+
+static _PyInitError
+preconfig_init_allocator(_PyPreConfig *config)
+{
     if (config->allocator == NULL) {
         /* bpo-34247. The PYTHONMALLOC environment variable has the priority
            over PYTHONDEV env var and "-X dev" command line option.
            For example, PYTHONMALLOC=malloc PYTHONDEVMODE=1 sets the memory
            allocators to "malloc" (and not to "debug"). */
-        const char *allocator = _PyPreConfig_GetEnv(config, "PYTHONMALLOC");
+        const char *allocator = _Py_GetEnv(config->use_environment, "PYTHONMALLOC");
         if (allocator) {
             config->allocator = _PyMem_RawStrdup(allocator);
             if (config->allocator == NULL) {
@@ -557,8 +635,47 @@ preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline)
             return _Py_INIT_NO_MEMORY();
         }
     }
+    return _Py_INIT_OK();
+}
+
+
+static _PyInitError
+preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline,
+               const _PyCoreConfig *coreconfig)
+{
+    _PyInitError err;
+
+    err = _PyPreCmdline_Read(cmdline, config, coreconfig);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    _PyPreCmdline_SetPreConfig(cmdline, config);
+
+    /* legacy_windows_fs_encoding, coerce_c_locale, utf8_mode */
+#ifdef MS_WINDOWS
+    _Py_get_env_flag(config->use_environment,
+                     &config->legacy_windows_fs_encoding,
+                     "PYTHONLEGACYWINDOWSFSENCODING");
+#endif
+
+    preconfig_init_coerce_c_locale(config);
+
+    err = preconfig_init_utf8_mode(config, cmdline);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    /* allocator */
+    err = preconfig_init_allocator(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
 
     assert(config->coerce_c_locale >= 0);
+#ifdef MS_WINDOWS
+    assert(config->legacy_windows_fs_encoding >= 0);
+#endif
     assert(config->utf8_mode >= 0);
     assert(config->isolated >= 0);
     assert(config->use_environment >= 0);
@@ -568,188 +685,15 @@ preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline)
 }
 
 
-static _PyInitError
-get_ctype_locale(char **locale_p)
-{
-    const char *loc = setlocale(LC_CTYPE, NULL);
-    if (loc == NULL) {
-        return _Py_INIT_ERR("failed to LC_CTYPE locale");
-    }
-
-    char *copy = _PyMem_RawStrdup(loc);
-    if (copy == NULL) {
-        return _Py_INIT_NO_MEMORY();
-    }
-
-    *locale_p = copy;
-    return _Py_INIT_OK();
-}
-
-
-PyObject*
-_PyPreConfig_AsDict(const _PyPreConfig *config)
-{
-    PyObject *dict;
-
-    dict = PyDict_New();
-    if (dict == NULL) {
-        return NULL;
-    }
-
-#define SET_ITEM(KEY, EXPR) \
-        do { \
-            PyObject *obj = (EXPR); \
-            if (obj == NULL) { \
-                goto fail; \
-            } \
-            int res = PyDict_SetItemString(dict, (KEY), obj); \
-            Py_DECREF(obj); \
-            if (res < 0) { \
-                goto fail; \
-            } \
-        } while (0)
-#define SET_ITEM_INT(ATTR) \
-    SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR))
-#define FROM_STRING(STR) \
-    ((STR != NULL) ? \
-        PyUnicode_FromString(STR) \
-        : (Py_INCREF(Py_None), Py_None))
-#define SET_ITEM_STR(ATTR) \
-    SET_ITEM(#ATTR, FROM_STRING(config->ATTR))
-
-    SET_ITEM_INT(isolated);
-    SET_ITEM_INT(use_environment);
-    SET_ITEM_INT(coerce_c_locale);
-    SET_ITEM_INT(coerce_c_locale_warn);
-    SET_ITEM_INT(utf8_mode);
-#ifdef MS_WINDOWS
-    SET_ITEM_INT(legacy_windows_fs_encoding);
-#endif
-    SET_ITEM_INT(dev_mode);
-    SET_ITEM_STR(allocator);
-    return dict;
-
-fail:
-    Py_DECREF(dict);
-    return NULL;
-
-#undef FROM_STRING
-#undef SET_ITEM
-#undef SET_ITEM_INT
-#undef SET_ITEM_STR
-}
-
-
-/* Parse the command line arguments */
-_PyInitError
-_PyPreCmdline_Read(_PyPreCmdline *cmdline)
-{
-    _PyWstrList *argv = &cmdline->argv;
-
-    _PyOS_ResetGetOpt();
-    /* Don't log parsing errors into stderr here: _PyCoreConfig_ReadFromArgv()
-       is responsible for that */
-    _PyOS_opterr = 0;
-    do {
-        int longindex = -1;
-        int c = _PyOS_GetOpt(argv->length, argv->items, &longindex);
-
-        if (c == EOF || c == 'c' || c == 'm') {
-            break;
-        }
-
-        switch (c) {
-        case 'E':
-            cmdline->use_environment = 0;
-            break;
-
-        case 'I':
-            cmdline->isolated = 1;
-            break;
-
-        case 'X':
-        {
-            if (_PyWstrList_Append(&cmdline->xoptions, _PyOS_optarg) < 0) {
-                return _Py_INIT_NO_MEMORY();
-            }
-            break;
-        }
-
-        default:
-            /* ignore other argument:
-               handled by _PyCoreConfig_ReadFromArgv() */
-            break;
-        }
-    } while (1);
-
-    return _Py_INIT_OK();
-}
-
-
-/* Read the configuration from:
-
-   - environment variables
-   - Py_xxx global configuration variables
-   - the LC_CTYPE locale
-
-   See _PyPreConfig_ReadFromArgv() to parse also command line arguments. */
-_PyInitError
-_PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args,
-                  const _PyCoreConfig *coreconfig)
-{
-    _PyInitError err;
-    _PyPreCmdline cmdline = _PyPreCmdline_INIT;
-    char *old_loc = NULL;
-
-    err = get_ctype_locale(&old_loc);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
-    }
-
-    /* Set LC_CTYPE to the user preferred locale */
-    _Py_SetLocaleFromEnv(LC_CTYPE);
-
-    _PyPreConfig_GetGlobalConfig(config);
-
-    _PyPreCmdline_GetPreConfig(&cmdline, config);
-
-    if (coreconfig) {
-        _PyPreCmdline_GetCoreConfig(&cmdline, coreconfig);
-        if (config->dev_mode == -1) {
-            config->dev_mode = coreconfig->dev_mode;
-        }
-    }
-
-    if (args) {
-        err = _PyPreCmdline_SetArgv(&cmdline, args);
-        if (_Py_INIT_FAILED(err)) {
-            goto done;
-        }
-    }
-
-    err = preconfig_read(config, &cmdline);
-
-done:
-    if (old_loc != NULL) {
-        setlocale(LC_CTYPE, old_loc);
-        PyMem_RawFree(old_loc);
-    }
-    _PyPreCmdline_Clear(&cmdline);
-
-    return err;
-}
-
-
 /* Read the configuration from:
 
    - command line arguments
    - environment variables
    - Py_xxx global configuration variables
-   - the LC_CTYPE locale
+   - the LC_CTYPE locale */
-
-   See _PyPreConfig_ReadFromArgv() to parse also command line arguments. */
 _PyInitError
-_PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args)
+_PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args,
+                  const _PyCoreConfig *coreconfig)
 {
     _PyInitError err;
 
@@ -758,30 +702,42 @@ _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args)
         return err;
     }
 
-    char *init_ctype_locale = NULL;
-    int init_utf8_mode = Py_UTF8Mode;
-#ifdef MS_WINDOWS
-    int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag;
-#endif
-    _PyPreConfig save_config = _PyPreConfig_INIT;
-    int locale_coerced = 0;
-    int loops = 0;
-
-    err = get_ctype_locale(&init_ctype_locale);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
-    }
-
     _PyPreConfig_GetGlobalConfig(config);
 
+    /* Copy LC_CTYPE locale, since it's modified later */
+    const char *loc = setlocale(LC_CTYPE, NULL);
+    if (loc == NULL) {
+        return _Py_INIT_ERR("failed to LC_CTYPE locale");
+    }
+    char *init_ctype_locale = _PyMem_RawStrdup(loc);
+    if (init_ctype_locale == NULL) {
+        return _Py_INIT_NO_MEMORY();
+    }
+
+    /* Save the config to be able to restore it if encodings change */
+    _PyPreConfig save_config = _PyPreConfig_INIT;
     if (_PyPreConfig_Copy(&save_config, config) < 0) {
-        err = _Py_INIT_NO_MEMORY();
+        return _Py_INIT_NO_MEMORY();
-        goto done;
     }
 
     /* Set LC_CTYPE to the user preferred locale */
     _Py_SetLocaleFromEnv(LC_CTYPE);
 
+    _PyPreCmdline cmdline = _PyPreCmdline_INIT;
+    if (args) {
+        err = _PyPreCmdline_SetArgv(&cmdline, args);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+    }
+
+    int init_utf8_mode = Py_UTF8Mode;
+#ifdef MS_WINDOWS
+    int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag;
+#endif
+    int locale_coerced = 0;
+    int loops = 0;
+
     while (1) {
         int utf8_mode = config->utf8_mode;
 
@@ -800,7 +756,7 @@ _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args)
         Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding;
 #endif
 
-        err = _PyPreConfig_Read(config, args, NULL);
+        err = preconfig_read(config, &cmdline, coreconfig);
         if (_Py_INIT_FAILED(err)) {
             goto done;
         }
@@ -864,6 +820,7 @@ done:
 #ifdef MS_WINDOWS
     Py_LegacyWindowsFSEncodingFlag = init_legacy_encoding;
 #endif
+    _PyPreCmdline_Clear(&cmdline);
     return err;
 }
 

Python/pylifecycle.c

@@ -482,7 +482,7 @@ _Py_Initialize_ReconfigureCore(PyInterpreterState **interp_p,
     }
     *interp_p = interp;
 
-    _PyCoreConfig_SetGlobalConfig(core_config);
+    _PyCoreConfig_Write(core_config);
 
     if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) {
         return _Py_INIT_ERR("failed to copy core config");
@@ -506,7 +506,7 @@ pycore_init_runtime(const _PyCoreConfig *core_config)
         return _Py_INIT_ERR("main interpreter already initialized");
     }
 
-    _PyCoreConfig_SetGlobalConfig(core_config);
+    _PyCoreConfig_Write(core_config);
 
     _PyInitError err = _PyRuntime_Initialize();
     if (_Py_INIT_FAILED(err)) {
@@ -801,7 +801,7 @@ pyinit_coreconfig(_PyCoreConfig *config, const _PyCoreConfig *src_config,
         return _Py_INIT_ERR("failed to copy core config");
     }
 
-    _PyInitError err = _PyCoreConfig_Read(config);
+    _PyInitError err = _PyCoreConfig_Read(config, NULL);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -833,14 +833,12 @@ pyinit_coreconfig(_PyCoreConfig *config, const _PyCoreConfig *src_config,
  * safe to call without calling Py_Initialize first)
  */
 _PyInitError
-_Py_InitializeCore(PyInterpreterState **interp_p,
+_Py_InitializeCore(const _PyCoreConfig *src_config,
-                   const _PyCoreConfig *src_config)
+                   PyInterpreterState **interp_p)
 {
-    _PyInitError err;
-
     assert(src_config != NULL);
 
-    err = _Py_PreInitializeFromConfig(src_config);
+    _PyInitError err = _Py_PreInitializeFromConfig(src_config);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -987,7 +985,7 @@ _Py_InitializeFromConfig(const _PyCoreConfig *config)
 {
     PyInterpreterState *interp = NULL;
     _PyInitError err;
-    err = _Py_InitializeCore(&interp, config);
+    err = _Py_InitializeCore(config, &interp);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }