bpo-36301: Cleanup preconfig code (GH-12535)

Prepare code to move some _PyPreConfig parameters into _PyPreCmdline.
Changes:

* _PyCoreConfig_ReadFromArgv(): remove preconfig parameter,
  use _PyRuntime.preconfig.
* Add _PyPreCmdline_GetPreConfig() (called by _PyPreConfig_Read()).
* Rename _PyPreCmdline_Init() to _PyPreCmdline_SetArgv()
* Factorize _Py_PreInitializeFromPreConfig() code: add
  pyinit_preinit().
* _PyPreConfig_Read() now sets coerce_c_locale to 2 if it must be
  coerced.
* Remove _PyCoreConfig_ReadPreConfig().
* _PyCoreConfig_Write() now copies updated preconfig into _PyRuntime.

Include/cpython/coreconfig.h

@@ -73,7 +73,12 @@ typedef struct {
        set to !Py_IgnoreEnvironmentFlag. */
     int use_environment;
 
-    int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
+    /* PYTHONCOERCECLOCALE, -1 means unknown.
+
+       If it is equal to 1, LC_CTYPE locale is read to decide it it should be
+       coerced or not (ex: PYTHONCOERCECLOCALE=1). Internally, it is set to 2
+       if the LC_CTYPE locale must be coerced. */
+    int coerce_c_locale;
     int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
 
 #ifdef MS_WINDOWS

Include/internal/pycore_coreconfig.h

@@ -25,12 +25,12 @@ typedef struct {
 /* Note: _PyPreCmdline_INIT sets other fields to 0/NULL */
 
 PyAPI_FUNC(void) _PyPreCmdline_Clear(_PyPreCmdline *cmdline);
-PyAPI_FUNC(_PyInitError) _PyPreCmdline_Init(_PyPreCmdline *cmdline,
+PyAPI_FUNC(_PyInitError) _PyPreCmdline_SetArgv(_PyPreCmdline *cmdline,
     const _PyArgv *args);
-PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline);
 PyAPI_FUNC(void) _PyPreCmdline_SetPreConfig(
     const _PyPreCmdline *cmdline,
     _PyPreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline);
 
 
 /* --- _PyWstrList ------------------------------------------------ */
@@ -76,7 +76,8 @@ PyAPI_FUNC(const char*) _PyPreConfig_GetEnv(const _PyPreConfig *config,
 PyAPI_FUNC(void) _Py_get_env_flag(_PyPreConfig *config,
     int *flag,
     const char *name);
-PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config);
+PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config,
+    const _PyArgv *args);
 PyAPI_FUNC(int) _PyPreConfig_AsDict(const _PyPreConfig *config,
     PyObject *dict);
 PyAPI_FUNC(_PyInitError) _PyPreConfig_ReadFromArgv(_PyPreConfig *config,
@@ -103,12 +104,10 @@ PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup(
     wchar_t **dest,
     wchar_t *wname,
     char *name);
-PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config,
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config);
-    const _PyPreConfig *preconfig);
 PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config,
-    const _PyArgv *args,
+    const _PyArgv *args);
-    const _PyPreConfig *preconfig);
+PyAPI_FUNC(_PyInitError) _PyCoreConfig_Write(const _PyCoreConfig *config);
-PyAPI_FUNC(void) _PyCoreConfig_Write(const _PyCoreConfig *config);
 
 #ifdef __cplusplus
 }

Modules/main.c

@@ -283,28 +283,38 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config,
 /* --- pymain_init() ---------------------------------------------- */
 
 static _PyInitError
-pymain_init_preconfig(_PyPreConfig *config, const _PyArgv *args)
+pymain_init_preconfig(const _PyArgv *args)
 {
-    _PyInitError err = _PyPreConfig_ReadFromArgv(config, args);
+    _PyInitError err;
+
+    _PyPreConfig config = _PyPreConfig_INIT;
+
+    err = _PyPreConfig_ReadFromArgv(&config, args);
     if (_Py_INIT_FAILED(err)) {
-        return err;
+        goto done;
     }
 
-    return _Py_PreInitializeFromPreConfig(config);
+    err = _Py_PreInitializeFromPreConfig(&config);
+
+done:
+    _PyPreConfig_Clear(&config);
+    return err;
 }
 
 
 static _PyInitError
 pymain_init_coreconfig(_PyCoreConfig *config, const _PyArgv *args,
-                       const _PyPreConfig *preconfig,
                        PyInterpreterState **interp_p)
 {
-    _PyInitError err = _PyCoreConfig_ReadFromArgv(config, args, preconfig);
+    _PyInitError err = _PyCoreConfig_ReadFromArgv(config, args);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
-    _PyCoreConfig_Write(config);
+    err = _PyCoreConfig_Write(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
 
     return _Py_InitializeCore(interp_p, config);
 }
@@ -348,18 +358,14 @@ pymain_init(const _PyArgv *args, PyInterpreterState **interp_p)
     fedisableexcept(FE_OVERFLOW);
 #endif
 
-    _PyPreConfig local_preconfig = _PyPreConfig_INIT;
+    err = pymain_init_preconfig(args);
-    _PyPreConfig *preconfig = &local_preconfig;
-
-    _PyCoreConfig local_config = _PyCoreConfig_INIT;
-    _PyCoreConfig *config = &local_config;
-
-    err = pymain_init_preconfig(preconfig, args);
     if (_Py_INIT_FAILED(err)) {
-        goto done;
+        return err;
     }
 
-    err = pymain_init_coreconfig(config, args, preconfig, interp_p);
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+
+    err = pymain_init_coreconfig(&config, args, interp_p);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
@@ -372,8 +378,7 @@ pymain_init(const _PyArgv *args, PyInterpreterState **interp_p)
     err = _Py_INIT_OK();
 
 done:
-    _PyPreConfig_Clear(preconfig);
+    _PyCoreConfig_Clear(&config);
-    _PyCoreConfig_Clear(config);
     return err;
 }
 

Python/coreconfig.c

@@ -1330,46 +1330,6 @@ config_init_fs_encoding(_PyCoreConfig *config)
 }
 
 
-static _PyInitError
-_PyCoreConfig_ReadPreConfig(_PyCoreConfig *config)
-{
-    _PyInitError err;
-    _PyPreConfig local_preconfig = _PyPreConfig_INIT;
-    _PyPreConfig_GetGlobalConfig(&local_preconfig);
-
-    if (_PyPreConfig_Copy(&local_preconfig, &config->preconfig) < 0) {
-        err = _Py_INIT_NO_MEMORY();
-        goto done;
-    }
-
-    err = _PyPreConfig_Read(&local_preconfig);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
-    }
-
-    if (_PyPreConfig_Copy(&config->preconfig, &local_preconfig) < 0) {
-        err = _Py_INIT_NO_MEMORY();
-        goto done;
-    }
-    err = _Py_INIT_OK();
-
-done:
-    _PyPreConfig_Clear(&local_preconfig);
-    return err;
-}
-
-
-static _PyInitError
-_PyCoreConfig_GetPreConfig(_PyCoreConfig *config)
-{
-    /* Read config written by _PyPreConfig_Write() */
-    if (_PyPreConfig_Copy(&config->preconfig, &_PyRuntime.preconfig) < 0) {
-        return _Py_INIT_NO_MEMORY();
-    }
-    return _Py_INIT_OK();
-}
-
-
 /* Read the configuration into _PyCoreConfig from:
 
    * Environment variables
@@ -1377,7 +1337,7 @@ _PyCoreConfig_GetPreConfig(_PyCoreConfig *config)
 
    See _PyCoreConfig_ReadFromArgv() to parse also command line arguments. */
 _PyInitError
-_PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig)
+_PyCoreConfig_Read(_PyCoreConfig *config)
 {
     _PyInitError err;
 
@@ -1386,25 +1346,12 @@ _PyCoreConfig_Read(_PyCoreConfig *config, const _PyPreConfig *preconfig)
         return err;
     }
 
-    err = _PyCoreConfig_GetPreConfig(config);
+    if (_PyPreConfig_Copy(&config->preconfig, &_PyRuntime.preconfig) < 0) {
-    if (_Py_INIT_FAILED(err)) {
+        return _Py_INIT_NO_MEMORY();
-        return err;
     }
 
     _PyCoreConfig_GetGlobalConfig(config);
 
-    if (preconfig != NULL) {
-        if (_PyPreConfig_Copy(&config->preconfig, preconfig) < 0) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-    else {
-        err = _PyCoreConfig_ReadPreConfig(config);
-        if (_Py_INIT_FAILED(err)) {
-            return err;
-        }
-    }
-
     assert(config->preconfig.use_environment >= 0);
 
     if (config->preconfig.isolated > 0) {
@@ -1548,11 +1495,22 @@ config_init_stdio(const _PyCoreConfig *config)
 
    - set Py_xxx global configuration variables
    - initialize C standard streams (stdin, stdout, stderr) */
-void
+_PyInitError
 _PyCoreConfig_Write(const _PyCoreConfig *config)
 {
     _PyCoreConfig_SetGlobalConfig(config);
     config_init_stdio(config);
+
+    /* Write the new pre-configuration into _PyRuntime */
+    PyMemAllocatorEx old_alloc;
+    _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    int res = _PyPreConfig_Copy(&_PyRuntime.preconfig, &config->preconfig);
+    PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
+    if (res < 0) {
+        return _Py_INIT_NO_MEMORY();
+    }
+
+    return _Py_INIT_OK();
 }
 
 
@@ -2047,8 +2005,7 @@ config_usage(int error, const wchar_t* program)
 
 /* Parse command line options and environment variables. */
 static _PyInitError
-config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
+config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline)
-                    const _PyPreConfig *preconfig)
 {
     int need_usage = 0;
     _PyInitError err;
@@ -2067,7 +2024,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
         return err;
     }
 
-    _PyPreCmdline_SetPreConfig(&cmdline->precmdline, &config->preconfig);
+    _PyPreCmdline_SetPreConfig(&cmdline->precmdline, &_PyRuntime.preconfig);
     if (_PyWstrList_Extend(&config->xoptions, &cmdline->precmdline.xoptions) < 0) {
         return _Py_INIT_NO_MEMORY();
     }
@@ -2098,7 +2055,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
         return err;
     }
 
-    err = _PyCoreConfig_Read(config, preconfig);
+    err = _PyCoreConfig_Read(config);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -2129,8 +2086,7 @@ config_from_cmdline(_PyCoreConfig *config, _PyCmdline *cmdline,
    * Environment variables
    * Py_xxx global configuration variables */
 _PyInitError
-_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args,
+_PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args)
-                           const _PyPreConfig *preconfig)
 {
     _PyInitError err;
 
@@ -2141,12 +2097,12 @@ _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args,
 
     _PyCmdline cmdline = {.precmdline = _PyPreCmdline_INIT};
 
-    err = _PyPreCmdline_Init(&cmdline.precmdline, args);
+    err = _PyPreCmdline_SetArgv(&cmdline.precmdline, args);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
-    err = config_from_cmdline(config, &cmdline, preconfig);
+    err = config_from_cmdline(config, &cmdline);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }

Python/pathconfig.c

@@ -394,7 +394,7 @@ pathconfig_global_init(void)
     _PyInitError err;
     _PyCoreConfig config = _PyCoreConfig_INIT;
 
-    err = _PyCoreConfig_Read(&config, NULL);
+    err = _PyCoreConfig_Read(&config);
     if (_Py_INIT_FAILED(err)) {
         goto error;
     }

Python/preconfig.c

@@ -111,12 +111,27 @@ _PyPreCmdline_Clear(_PyPreCmdline *cmdline)
 
 
 _PyInitError
-_PyPreCmdline_Init(_PyPreCmdline *cmdline, const _PyArgv *args)
+_PyPreCmdline_SetArgv(_PyPreCmdline *cmdline, const _PyArgv *args)
 {
     return _PyArgv_AsWstrList(args, &cmdline->argv);
 }
 
 
+static void
+_PyPreCmdline_GetPreConfig(_PyPreCmdline *cmdline, const _PyPreConfig *config)
+{
+#define COPY_ATTR(ATTR) \
+    if (config->ATTR != -1) { \
+        cmdline->ATTR = config->ATTR; \
+    }
+
+    COPY_ATTR(use_environment);
+    COPY_ATTR(isolated);
+
+#undef COPY_ATTR
+}
+
+
 /* --- _PyPreConfig ----------------------------------------------- */
 
 void
@@ -336,24 +351,28 @@ preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline)
 static void
 preconfig_init_locale(_PyPreConfig *config)
 {
-    /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't
+    /* The C locale enables the C locale coercion (PEP 538) */
-       imply that the C locale is always coerced. It is only coerced if
+    if (_Py_LegacyLocaleDetected()) {
-       if the LC_CTYPE locale is "C". */
+        config->coerce_c_locale = 2;
-    if (config->coerce_c_locale != 0) {
+    }
-        /* The C locale enables the C locale coercion (PEP 538) */
+    else {
-        if (_Py_LegacyLocaleDetected()) {
+        config->coerce_c_locale = 0;
-            config->coerce_c_locale = 1;
-        }
-        else {
-            config->coerce_c_locale = 0;
-        }
     }
 }
 
 
 static _PyInitError
-preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
+preconfig_read(_PyPreConfig *config, _PyPreCmdline *cmdline)
 {
+    _PyInitError err;
+
+    err = _PyPreCmdline_Read(cmdline);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    _PyPreCmdline_SetPreConfig(cmdline, config);
+
     _PyPreConfig_GetGlobalConfig(config);
 
     /* isolated and use_environment */
@@ -398,13 +417,16 @@ preconfig_read(_PyPreConfig *config, const _PyPreCmdline *cmdline)
 #endif
 
     if (config->utf8_mode < 0) {
-        _PyInitError err = preconfig_init_utf8_mode(config, cmdline);
+        err = preconfig_init_utf8_mode(config, cmdline);
         if (_Py_INIT_FAILED(err)) {
             return err;
         }
     }
 
-    if (config->coerce_c_locale != 0) {
+    /* Test also if coerce_c_locale equals 1: PYTHONCOERCECLOCALE=1 doesn't
+       imply that the C locale is always coerced. It is only coerced if
+       if the LC_CTYPE locale is "C". */
+    if (config->coerce_c_locale != 0 && config->coerce_c_locale != 2) {
         preconfig_init_locale(config);
     }
 
@@ -491,36 +513,6 @@ get_ctype_locale(char **locale_p)
 }
 
 
-/* Read the configuration from:
-
-   - environment variables
-   - Py_xxx global configuration variables
-   - the LC_CTYPE locale
-
-   See _PyPreConfig_ReadFromArgv() to parse also command line arguments. */
-_PyInitError
-_PyPreConfig_Read(_PyPreConfig *config)
-{
-    _PyInitError err;
-    char *old_loc;
-
-    err = get_ctype_locale(&old_loc);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    /* Set LC_CTYPE to the user preferred locale */
-    _Py_SetLocaleFromEnv(LC_CTYPE);
-
-    err = preconfig_read(config, NULL);
-
-    setlocale(LC_CTYPE, old_loc);
-    PyMem_RawFree(old_loc);
-
-    return err;
-}
-
-
 void
 _PyPreCmdline_SetPreConfig(const _PyPreCmdline *cmdline, _PyPreConfig *config)
 {
@@ -628,33 +620,46 @@ _PyPreCmdline_Read(_PyPreCmdline *cmdline)
 }
 
 
-static _PyInitError
+/* Read the configuration from:
-preconfig_from_argv(_PyPreConfig *config, const _PyArgv *args)
+
+   - environment variables
+   - Py_xxx global configuration variables
+   - the LC_CTYPE locale
+
+   See _PyPreConfig_ReadFromArgv() to parse also command line arguments. */
+_PyInitError
+_PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args)
 {
     _PyInitError err;
-
     _PyPreCmdline cmdline = _PyPreCmdline_INIT;
+    char *old_loc = NULL;
 
-    err = _PyPreCmdline_Init(&cmdline, args);
+    err = get_ctype_locale(&old_loc);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }
 
-    err = _PyPreCmdline_Read(&cmdline);
+    /* Set LC_CTYPE to the user preferred locale */
-    if (_Py_INIT_FAILED(err)) {
+    _Py_SetLocaleFromEnv(LC_CTYPE);
-        goto done;
-    }
 
-    _PyPreCmdline_SetPreConfig(&cmdline, config);
+    _PyPreCmdline_GetPreConfig(&cmdline, config);
+
+    if (args) {
+        err = _PyPreCmdline_SetArgv(&cmdline, args);
+        if (_Py_INIT_FAILED(err)) {
+            goto done;
+        }
+    }
 
     err = preconfig_read(config, &cmdline);
-    if (_Py_INIT_FAILED(err)) {
-        goto done;
-    }
-    err = _Py_INIT_OK();
 
 done:
+    if (old_loc != NULL) {
+        setlocale(LC_CTYPE, old_loc);
+        PyMem_RawFree(old_loc);
+    }
     _PyPreCmdline_Clear(&cmdline);
+
     return err;
 }
 
@@ -719,15 +724,11 @@ _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args)
         Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding;
 #endif
 
-        err = preconfig_from_argv(config, args);
+        err = _PyPreConfig_Read(config, args);
         if (_Py_INIT_FAILED(err)) {
             goto done;
         }
 
-        if (locale_coerced) {
-            config->coerce_c_locale = 1;
-        }
-
         /* The legacy C locale assumes ASCII as the default text encoding, which
          * causes problems not only for the CPython runtime, but also other
          * components like GNU readline.

Python/pylifecycle.c

@@ -714,23 +714,8 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p,
 }
 
 
-_PyInitError
-_Py_PreInitializeFromPreConfig(_PyPreConfig *config)
-{
-    if (config != NULL) {
-        _PyInitError err = _PyPreConfig_Write(config);
-        if (_Py_INIT_FAILED(err)) {
-            return err;
-        }
-    }
-
-    _PyRuntime.pre_initialized = 1;
-    return _Py_INIT_OK();
-}
-
-
 static _PyInitError
-pyinit_preconfig(_PyPreConfig *config, const _PyPreConfig *src_config)
+pyinit_preinit(_PyPreConfig *config, const _PyPreConfig *src_config)
 {
     _PyInitError err;
 
@@ -739,32 +724,46 @@ pyinit_preconfig(_PyPreConfig *config, const _PyPreConfig *src_config)
         return err;
     }
 
-    if (_PyPreConfig_Copy(config, src_config) < 0) {
+    if (_PyRuntime.pre_initialized) {
-        return _Py_INIT_ERR("failed to copy pre config");
+        /* If it's already configured: ignored the new configuration */
+        return _Py_INIT_OK();
     }
 
-    err = _PyPreConfig_Read(config);
+    if (src_config) {
+        if (_PyPreConfig_Copy(config, src_config) < 0) {
+            return _Py_INIT_ERR("failed to copy pre config");
+        }
+    }
+
+    err = _PyPreConfig_Read(config, NULL);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
 
-    return _Py_PreInitializeFromPreConfig(config);
+    err = _PyPreConfig_Write(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    _PyRuntime.pre_initialized = 1;
+    return _Py_INIT_OK();
+}
+
+
+_PyInitError
+_Py_PreInitializeFromPreConfig(_PyPreConfig *config)
+{
+    return pyinit_preinit(config, NULL);
 }
 
 
 _PyInitError
 _Py_PreInitialize(void)
 {
-    _PyInitError err = _PyRuntime_Initialize();
+    _PyPreConfig config = _PyPreConfig_INIT;
-    if (_Py_INIT_FAILED(err)) {
+    _PyInitError err = pyinit_preinit(&config, NULL);
-        return err;
+    _PyPreConfig_Clear(&config);
-    }
+    return err;
-
-    if (_PyRuntime.pre_initialized) {
-        return _Py_INIT_OK();
-    }
-
-    return _Py_PreInitializeFromPreConfig(NULL);
 }
 
 
@@ -776,7 +775,7 @@ pyinit_coreconfig(_PyCoreConfig *config, const _PyCoreConfig *src_config,
         return _Py_INIT_ERR("failed to copy core config");
     }
 
-    _PyInitError err = _PyCoreConfig_Read(config, NULL);
+    _PyInitError err = _PyCoreConfig_Read(config);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -817,7 +816,7 @@ _Py_InitializeCore(PyInterpreterState **interp_p,
 
     _PyCoreConfig local_config = _PyCoreConfig_INIT;
 
-    err = pyinit_preconfig(&local_config.preconfig, &src_config->preconfig);
+    err = pyinit_preinit(&local_config.preconfig, &src_config->preconfig);
     if (_Py_INIT_FAILED(err)) {
         goto done;
     }