bpo-32030: Add _PyMainInterpreterConfig_ReadEnv() (#4542)

Py_GetPath() and Py_Main() now call
_PyMainInterpreterConfig_ReadEnv() to share the same code to get
environment variables.

Changes:

* Add _PyMainInterpreterConfig_ReadEnv()
* Add _PyMainInterpreterConfig_Clear()
* Add _PyMem_RawWcsdup()
* _PyMainInterpreterConfig: rename pythonhome to home
* Rename _Py_ReadMainInterpreterConfig() to
  _PyMainInterpreterConfig_Read()
* Use _Py_INIT_USER_ERR(), instead of _Py_INIT_ERR(), for decoding
  errors: the user is able to fix the issue, it's not a bug in
  Python. Same change was made in _Py_INIT_NO_MEMORY().
* Remove _Py_GetPythonHomeWithConfig()

Include/pylifecycle.h

@@ -30,7 +30,7 @@ typedef struct {
    Don't abort() the process on such error. */
 #define _Py_INIT_USER_ERR(MSG) \
     (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1}
-#define _Py_INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed")
+#define _Py_INIT_NO_MEMORY() _Py_INIT_USER_ERR("memory allocation failed")
 #define _Py_INIT_FAILED(err) \
     (err.msg != NULL)
 
@@ -42,11 +42,6 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
 
 PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
 PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
-#ifdef Py_BUILD_CORE
-PyAPI_FUNC(_PyInitError) _Py_GetPythonHomeWithConfig(
-    const _PyMainInterpreterConfig *config,
-    wchar_t **home);
-#endif
 
 #ifndef Py_LIMITED_API
 /* Only used by applications that embed the interpreter and need to
@@ -58,7 +53,11 @@ PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
 /* PEP 432 Multi-phase initialization API (Private while provisional!) */
 PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *);
 PyAPI_FUNC(int) _Py_IsCoreInitialized(void);
-PyAPI_FUNC(_PyInitError) _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *);
+
+PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *);
+PyAPI_FUNC(_PyInitError) _PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *);
+PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *);
+
 PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *);
 #endif
 

Include/pymem.h

@@ -105,8 +105,14 @@ PyAPI_FUNC(void *) PyMem_Realloc(void *ptr, size_t new_size);
 PyAPI_FUNC(void) PyMem_Free(void *ptr);
 
 #ifndef Py_LIMITED_API
+/* strdup() using PyMem_RawMalloc() */
 PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str);
+
+/* strdup() using PyMem_Malloc() */
 PyAPI_FUNC(char *) _PyMem_Strdup(const char *str);
+
+/* wcsdup() using PyMem_RawMalloc() */
+PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str);
 #endif
 
 /* Macros. */

Include/pystate.h

@@ -60,16 +60,17 @@ typedef struct {
  */
 typedef struct {
     int install_signal_handlers;
-    wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
+    /* PYTHONPATH environment variable */
-    wchar_t *pythonhome;    /* PYTHONHOME environment variable,
+    wchar_t *module_search_path_env;
-                               see also Py_SetPythonHome(). */
+    /* PYTHONHOME environment variable, see also Py_SetPythonHome(). */
+    wchar_t *home;
 } _PyMainInterpreterConfig;
 
 #define _PyMainInterpreterConfig_INIT \
     (_PyMainInterpreterConfig){\
      .install_signal_handlers = -1, \
      .module_search_path_env = NULL, \
-     .pythonhome = NULL}
+     .home = NULL}
 
 typedef struct _is {
 

Modules/getpath.c

@@ -120,7 +120,6 @@ typedef struct {
     wchar_t *path_env;                 /* PATH environment variable */
     wchar_t *home;                     /* PYTHONHOME environment variable */
     wchar_t *module_search_path_env;   /* PYTHONPATH environment variable */
-    wchar_t *module_search_path_buffer;
 
     wchar_t *prog;                     /* Program name */
     wchar_t *pythonpath;               /* PYTHONPATH define */
@@ -886,9 +885,9 @@ calculate_module_search_path(PyCalculatePath *calculate, PyPathConfig *config)
 }
 
 
-#define DECODE_FAILED(NAME, LEN) \
+#define DECODE_LOCALE_ERR(NAME, LEN) \
     ((LEN) == (size_t)-2) \
-     ? _Py_INIT_ERR("failed to decode " #NAME) \
+     ? _Py_INIT_USER_ERR("failed to decode " #NAME) \
      : _Py_INIT_NO_MEMORY()
 
 
@@ -896,19 +895,15 @@ static _PyInitError
 calculate_init(PyCalculatePath *calculate,
                const _PyMainInterpreterConfig *main_config)
 {
-    _PyInitError err;
+    calculate->home = main_config->home;
-
+    calculate->module_search_path_env = main_config->module_search_path_env;
-    err = _Py_GetPythonHomeWithConfig(main_config, &calculate->home);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
 
     size_t len;
     char *path = getenv("PATH");
     if (path) {
         calculate->path_env = Py_DecodeLocale(path, &len);
         if (!calculate->path_env) {
-            return DECODE_FAILED("PATH environment variable", len);
+            return DECODE_LOCALE_ERR("PATH environment variable", len);
         }
     }
 
@@ -916,37 +911,19 @@ calculate_init(PyCalculatePath *calculate,
 
     calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len);
     if (!calculate->pythonpath) {
-        return DECODE_FAILED("PYTHONPATH define", len);
+        return DECODE_LOCALE_ERR("PYTHONPATH define", len);
     }
     calculate->prefix = Py_DecodeLocale(PREFIX, &len);
     if (!calculate->prefix) {
-        return DECODE_FAILED("PREFIX define", len);
+        return DECODE_LOCALE_ERR("PREFIX define", len);
     }
     calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len);
     if (!calculate->prefix) {
-        return DECODE_FAILED("EXEC_PREFIX define", len);
+        return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
     }
     calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len);
     if (!calculate->lib_python) {
-        return DECODE_FAILED("EXEC_PREFIX define", len);
+        return DECODE_LOCALE_ERR("EXEC_PREFIX define", len);
-    }
-
-    calculate->module_search_path_env = NULL;
-    if (main_config) {
-        if (main_config->module_search_path_env) {
-            calculate->module_search_path_env = main_config->module_search_path_env;
-        }
-
-    }
-    else {
-        char *pythonpath = Py_GETENV("PYTHONPATH");
-        if (pythonpath && pythonpath[0] != '\0') {
-            calculate->module_search_path_buffer = Py_DecodeLocale(pythonpath, &len);
-            if (!calculate->module_search_path_buffer) {
-                return DECODE_FAILED("PYTHONPATH environment variable", len);
-            }
-            calculate->module_search_path_env = calculate->module_search_path_buffer;
-        }
     }
     return _Py_INIT_OK();
 }
@@ -960,7 +937,6 @@ calculate_free(PyCalculatePath *calculate)
     PyMem_RawFree(calculate->exec_prefix);
     PyMem_RawFree(calculate->lib_python);
     PyMem_RawFree(calculate->path_env);
-    PyMem_RawFree(calculate->module_search_path_buffer);
 }
 
 
@@ -988,13 +964,24 @@ calculate_path_impl(PyCalculatePath *calculate, PyPathConfig *config)
 static void
 calculate_path(const _PyMainInterpreterConfig *main_config)
 {
+    _PyInitError err;
     PyCalculatePath calculate;
     memset(&calculate, 0, sizeof(calculate));
 
-    _PyInitError err = calculate_init(&calculate, main_config);
+    _PyMainInterpreterConfig tmp_config;
+    int use_tmp = (main_config == NULL);
+    if (use_tmp) {
+        tmp_config = _PyMainInterpreterConfig_INIT;
+        err = _PyMainInterpreterConfig_ReadEnv(&tmp_config);
         if (_Py_INIT_FAILED(err)) {
-        calculate_free(&calculate);
+            goto fatal_error;
-        _Py_FatalInitError(err);
+        }
+        main_config = &tmp_config;
+    }
+
+    err = calculate_init(&calculate, main_config);
+    if (_Py_INIT_FAILED(err)) {
+        goto fatal_error;
     }
 
     PyPathConfig new_path_config;
@@ -1003,7 +990,18 @@ calculate_path(const _PyMainInterpreterConfig *main_config)
     calculate_path_impl(&calculate, &new_path_config);
     path_config = new_path_config;
 
+    if (use_tmp) {
+        _PyMainInterpreterConfig_Clear(&tmp_config);
+    }
     calculate_free(&calculate);
+    return;
+
+fatal_error:
+    if (use_tmp) {
+        _PyMainInterpreterConfig_Clear(&tmp_config);
+    }
+    calculate_free(&calculate);
+    _Py_FatalInitError(err);
 }
 
 

Modules/main.c

@@ -36,10 +36,16 @@
 extern "C" {
 #endif
 
+#define DECODE_LOCALE_ERR(NAME, LEN) \
+    (((LEN) == -2) \
+     ? _Py_INIT_USER_ERR("failed to decode " #NAME) \
+     : _Py_INIT_NO_MEMORY())
+
+
 #define SET_DECODE_ERROR(NAME, LEN) \
     do { \
         if ((LEN) == (size_t)-2) { \
-            pymain->err = _Py_INIT_ERR("failed to decode " #NAME); \
+            pymain->err = _Py_INIT_USER_ERR("failed to decode " #NAME); \
         } \
         else { \
             pymain->err = _Py_INIT_NO_MEMORY(); \
@@ -450,7 +456,7 @@ pymain_free_impl(_PyMain *pymain)
     Py_CLEAR(pymain->main_importer_path);
     PyMem_RawFree(pymain->program_name);
 
-    PyMem_RawFree(pymain->config.module_search_path_env);
+    _PyMainInterpreterConfig_Clear(&pymain->config);
 
 #ifdef __INSURE__
     /* Insure++ is a memory analysis tool that aids in discovering
@@ -515,20 +521,11 @@ error:
 static wchar_t*
 pymain_wstrdup(_PyMain *pymain, wchar_t *str)
 {
-    size_t len = wcslen(str) + 1;  /* +1 for NUL character */
+    wchar_t *str2 = _PyMem_RawWcsdup(str);
-    if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
-        pymain->err = _Py_INIT_NO_MEMORY();
-        return NULL;
-    }
-
-    size_t size = len * sizeof(wchar_t);
-    wchar_t *str2 = PyMem_RawMalloc(size);
     if (str2 == NULL) {
         pymain->err = _Py_INIT_NO_MEMORY();
         return NULL;
     }
-
-    memcpy(str2, str, size);
     return str2;
 }
 
@@ -955,7 +952,7 @@ pymain_init_main_interpreter(_PyMain *pymain)
     _PyInitError err;
 
     /* TODO: Print any exceptions raised by these operations */
-    err = _Py_ReadMainInterpreterConfig(&pymain->config);
+    err = _PyMainInterpreterConfig_Read(&pymain->config);
     if (_Py_INIT_FAILED(err)) {
         pymain->err = err;
         return -1;
@@ -1361,8 +1358,7 @@ pymain_set_flags_from_env(_PyMain *pymain)
 
 
 static int
-pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest,
+config_get_env_var_dup(wchar_t **dest, wchar_t *wname, char *name)
-                       wchar_t *wname, char *name)
 {
     if (Py_IgnoreEnvironmentFlag) {
         *dest = NULL;
@@ -1376,7 +1372,7 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest,
         return 0;
     }
 
-    wchar_t *copy = pymain_wstrdup(pymain, var);
+    wchar_t *copy = _PyMem_RawWcsdup(var);
     if (copy == NULL) {
         return -1;
     }
@@ -1393,11 +1389,9 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest,
     wchar_t *wvar = Py_DecodeLocale(var, &len);
     if (!wvar) {
         if (len == (size_t)-2) {
-            /* don't set pymain->err */
             return -2;
         }
         else {
-            pymain->err = _Py_INIT_NO_MEMORY();
             return -1;
         }
     }
@@ -1407,25 +1401,21 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest,
 }
 
 
-static int
+static _PyInitError
-pymain_init_pythonpath(_PyMain *pymain)
+config_init_pythonpath(_PyMainInterpreterConfig *config)
 {
     wchar_t *path;
-    int res = pymain_get_env_var_dup(pymain, &path,
+    int res = config_get_env_var_dup(&path, L"PYTHONPATH", "PYTHONPATH");
-                                     L"PYTHONPATH", "PYTHONPATH");
     if (res < 0) {
-        if (res == -2) {
+        return DECODE_LOCALE_ERR("PYTHONHOME", res);
-            SET_DECODE_ERROR("PYTHONPATH", (size_t)-2);
     }
-        return -1;
+    config->module_search_path_env = path;
-    }
+    return _Py_INIT_OK();
-    pymain->config.module_search_path_env = path;
-    return 0;
 }
 
 
-static int
+static _PyInitError
-pymain_init_pythonhome(_PyMain *pymain)
+config_init_pythonhome(_PyMainInterpreterConfig *config)
 {
     wchar_t *home;
 
@@ -1433,26 +1423,41 @@ pymain_init_pythonhome(_PyMain *pymain)
     if (home) {
         /* Py_SetPythonHome() has been called before Py_Main(),
            use its value */
-        pymain->config.pythonhome = pymain_wstrdup(pymain, home);
+        config->home = _PyMem_RawWcsdup(home);
-        if (pymain->config.pythonhome == NULL) {
+        if (config->home == NULL) {
-            return -1;
+            return _Py_INIT_NO_MEMORY();
         }
-        return 0;
+        return _Py_INIT_OK();
     }
 
-    int res = pymain_get_env_var_dup(pymain, &home,
+    int res = config_get_env_var_dup(&home, L"PYTHONHOME", "PYTHONHOME");
-                                     L"PYTHONHOME", "PYTHONHOME");
     if (res < 0) {
-        if (res == -2) {
+        return DECODE_LOCALE_ERR("PYTHONHOME", res);
-            SET_DECODE_ERROR("PYTHONHOME", (size_t)-2);
     }
-        return -1;
+    config->home = home;
-    }
+    return _Py_INIT_OK();
-    pymain->config.pythonhome = home;
-    return 0;
 }
 
 
+_PyInitError
+_PyMainInterpreterConfig_ReadEnv(_PyMainInterpreterConfig *config)
+{
+    _PyInitError err = config_init_pythonhome(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    err = config_init_pythonpath(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    return _Py_INIT_OK();
+}
+
+
+
+
 static int
 pymain_parse_envvars(_PyMain *pymain)
 {
@@ -1474,10 +1479,10 @@ pymain_parse_envvars(_PyMain *pymain)
         return -1;
     }
     core_config->allocator = Py_GETENV("PYTHONMALLOC");
-    if (pymain_init_pythonpath(pymain) < 0) {
+
-        return -1;
+    _PyInitError err = _PyMainInterpreterConfig_ReadEnv(&pymain->config);
-    }
+    if (_Py_INIT_FAILED(pymain->err)) {
-    if (pymain_init_pythonhome(pymain) < 0) {
+        pymain->err = err;
         return -1;
     }
 

Objects/obmalloc.c

@@ -455,6 +455,24 @@ PyMem_Free(void *ptr)
 }
 
 
+wchar_t*
+_PyMem_RawWcsdup(const wchar_t *str)
+{
+    size_t len = wcslen(str);
+    if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) {
+        return NULL;
+    }
+
+    size_t size = (len + 1) * sizeof(wchar_t);
+    wchar_t *str2 = PyMem_RawMalloc(size);
+    if (str2 == NULL) {
+        return NULL;
+    }
+
+    memcpy(str2, str, size);
+    return str2;
+}
+
 char *
 _PyMem_RawStrdup(const char *str)
 {

PC/getpathp.c

@@ -689,26 +689,12 @@ error:
 }
 
 
-static _PyInitError
+static void
 calculate_init(PyCalculatePath *calculate,
                const _PyMainInterpreterConfig *main_config)
 {
-    _PyInitError err;
+    calculate->home = main_config->home;
-
-    err = _Py_GetPythonHomeWithConfig(main_config, &calculate->home);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
-    if (main_config) {
     calculate->module_search_path_env = main_config->module_search_path_env;
-    }
-    else if (!Py_IgnoreEnvironmentFlag) {
-        wchar_t *path = _wgetenv(L"PYTHONPATH");
-        if (path && *path != '\0') {
-            calculate->module_search_path_env = path;
-        }
-    }
 
     calculate->path_env = _wgetenv(L"PATH");
 
@@ -717,8 +703,6 @@ calculate_init(PyCalculatePath *calculate,
         prog = L"python";
     }
     calculate->prog = prog;
-
-    return _Py_INIT_OK();
 }
 
 
@@ -1020,14 +1004,22 @@ calculate_free(PyCalculatePath *calculate)
 static void
 calculate_path(const _PyMainInterpreterConfig *main_config)
 {
+    _PyInitError err;
     PyCalculatePath calculate;
     memset(&calculate, 0, sizeof(calculate));
 
-    _PyInitError err = calculate_init(&calculate, main_config);
+    _PyMainInterpreterConfig tmp_config;
+    int use_tmp = (main_config == NULL);
+    if (use_tmp) {
+        tmp_config = _PyMainInterpreterConfig_INIT;
+        err = _PyMainInterpreterConfig_ReadEnv(&tmp_config);
         if (_Py_INIT_FAILED(err)) {
-        calculate_free(&calculate);
+            goto fatal_error;
-        _Py_FatalInitError(err);
         }
+        main_config = &tmp_config;
+    }
+
+    calculate_init(&calculate, main_config);
 
     PyPathConfig new_path_config;
     memset(&new_path_config, 0, sizeof(new_path_config));
@@ -1035,7 +1027,18 @@ calculate_path(const _PyMainInterpreterConfig *main_config)
     calculate_path_impl(&calculate, &new_path_config, main_config);
     path_config = new_path_config;
 
+    if (use_tmp) {
+        _PyMainInterpreterConfig_Clear(&tmp_config);
+    }
     calculate_free(&calculate);
+    return;
+
+fatal_error:
+    if (use_tmp) {
+        _PyMainInterpreterConfig_Clear(&tmp_config);
+    }
+    calculate_free(&calculate);
+    _Py_FatalInitError(err);
 }
 
 

Python/pylifecycle.c

@@ -796,7 +796,7 @@ _Py_InitializeCore(const _PyCoreConfig *config)
  */
 
 _PyInitError
-_Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config)
+_PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *config)
 {
     /* Signal handlers are installed by default */
     if (config->install_signal_handlers < 0) {
@@ -805,6 +805,17 @@ _Py_ReadMainInterpreterConfig(_PyMainInterpreterConfig *config)
     return _Py_INIT_OK();
 }
 
+
+void
+_PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *config)
+{
+    PyMem_RawFree(config->module_search_path_env);
+    config->module_search_path_env = NULL;
+    PyMem_RawFree(config->home);
+    config->home = NULL;
+}
+
+
 /* Update interpreter state based on supplied configuration settings
  *
  * After calling this function, most of the restrictions on the interpreter
@@ -943,7 +954,7 @@ _Py_InitializeEx_Private(int install_sigs, int install_importlib)
     }
 
     /* TODO: Print any exceptions raised by these operations */
-    err = _Py_ReadMainInterpreterConfig(&config);
+    err = _PyMainInterpreterConfig_Read(&config);
     if (_Py_INIT_FAILED(err)) {
         return err;
     }
@@ -1477,8 +1488,8 @@ Py_SetPythonHome(wchar_t *home)
 }
 
 
-_PyInitError
+wchar_t*
-_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config, wchar_t **homep)
+Py_GetPythonHome(void)
 {
     /* Use a static buffer to avoid heap memory allocation failure.
        Py_GetPythonHome() doesn't allow to report error, and the caller
@@ -1486,40 +1497,22 @@ _Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config, wchar_t **ho
     static wchar_t buffer[MAXPATHLEN+1];
 
     if (default_home) {
-        *homep = default_home;
+        return default_home;
-        return _Py_INIT_OK();
-    }
-
-    if (config) {
-        *homep = config->pythonhome;
-        return _Py_INIT_OK();
     }
 
     char *home = Py_GETENV("PYTHONHOME");
     if (!home) {
-        *homep = NULL;
+        return NULL;
-        return _Py_INIT_OK();
     }
 
     size_t size = Py_ARRAY_LENGTH(buffer);
     size_t r = mbstowcs(buffer, home, size);
     if (r == (size_t)-1 || r >= size) {
         /* conversion failed or the static buffer is too small */
-        *homep = NULL;
+        return NULL;
-        return _Py_INIT_ERR("failed to decode PYTHONHOME environment variable");
     }
 
-    *homep = buffer;
+    return buffer;
-    return _Py_INIT_OK();
-}
-
-wchar_t *
-Py_GetPythonHome(void)
-{
-    wchar_t *home;
-    /* Ignore error */
-    (void)_Py_GetPythonHomeWithConfig(NULL, &home);
-    return home;
 }
 
 /* Add the __main__ module */