bpo-32030: Rewrite calculate_path() (#4521)

* calculate_path() rewritten in Modules/getpath.c and PC/getpathp.c
* Move global variables into a new PyPathConfig structure.
* calculate_path():

  * Split the huge calculate_path() function into subfunctions.
  * Add PyCalculatePath structure to pass data between subfunctions.
  * Document PyCalculatePath fields.
  * Move cleanup code into a new calculate_free() subfunction
  * calculate_init() now handles Py_DecodeLocale() failures properly
  * calculate_path() is now atomic: only replace PyPathConfig
    (path_config) at once on success.

* _Py_GetPythonHomeWithConfig() now returns an error on failure
* Add _Py_INIT_NO_MEMORY() helper: report a memory allocation failure
* Coding style fixes (PEP 7)

Include/pylifecycle.h

@@ -7,23 +7,7 @@
 extern "C" {
 #endif
 
-PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
-PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
-
-PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
-PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
-#ifdef Py_BUILD_CORE
-PyAPI_FUNC(wchar_t *) _Py_GetPythonHomeWithConfig(
-    const _PyMainInterpreterConfig *config);
-#endif
-
 #ifndef Py_LIMITED_API
-/* Only used by applications that embed the interpreter and need to
- * override the standard encoding determination mechanism
- */
-PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
-                                             const char *errors);
-
 typedef struct {
     const char *prefix;
     const char *msg;
@@ -46,9 +30,31 @@ typedef struct {
    Don't abort() the process on such error. */
 #define _Py_INIT_USER_ERR(MSG) \
     (_PyInitError){.prefix = _Py_INIT_GET_FUNC(), .msg = (MSG), .user_err = 1}
+#define _Py_INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed")
 #define _Py_INIT_FAILED(err) \
     (err.msg != NULL)
 
+#endif
+
+
+PyAPI_FUNC(void) Py_SetProgramName(wchar_t *);
+PyAPI_FUNC(wchar_t *) Py_GetProgramName(void);
+
+PyAPI_FUNC(void) Py_SetPythonHome(wchar_t *);
+PyAPI_FUNC(wchar_t *) Py_GetPythonHome(void);
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(_PyInitError) _Py_GetPythonHomeWithConfig(
+    const _PyMainInterpreterConfig *config,
+    wchar_t **home);
+#endif
+
+#ifndef Py_LIMITED_API
+/* Only used by applications that embed the interpreter and need to
+ * override the standard encoding determination mechanism
+ */
+PyAPI_FUNC(int) Py_SetStandardStreamEncoding(const char *encoding,
+                                             const char *errors);
+
 /* PEP 432 Multi-phase initialization API (Private while provisional!) */
 PyAPI_FUNC(_PyInitError) _Py_InitializeCore(const _PyCoreConfig *);
 PyAPI_FUNC(int) _Py_IsCoreInitialized(void);

Modules/main.c

@@ -36,6 +36,16 @@
 extern "C" {
 #endif
 
+#define SET_DECODE_ERROR(NAME, LEN) \
+    do { \
+        if ((LEN) == (size_t)-2) { \
+            pymain->err = _Py_INIT_ERR("failed to decode " #NAME); \
+        } \
+        else { \
+            pymain->err = _Py_INIT_NO_MEMORY(); \
+        } \
+    } while (0)
+
 /* For Py_GetArgcArgv(); set by main() */
 static wchar_t **orig_argv;
 static int orig_argc;
@@ -417,9 +427,6 @@ typedef struct {
      .env_warning_options = {0, NULL}}
 
 
-#define INIT_NO_MEMORY() _Py_INIT_ERR("memory allocation failed")
-
-
 static void
 pymain_optlist_clear(_Py_OptList *list)
 {
@@ -510,14 +517,14 @@ pymain_wstrdup(_PyMain *pymain, wchar_t *str)
 {
     size_t len = wcslen(str) + 1;  /* +1 for NUL character */
     if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
-        pymain->err = INIT_NO_MEMORY();
+        pymain->err = _Py_INIT_NO_MEMORY();
         return NULL;
     }
 
     size_t size = len * sizeof(wchar_t);
     wchar_t *str2 = PyMem_RawMalloc(size);
     if (str2 == NULL) {
-        pymain->err = INIT_NO_MEMORY();
+        pymain->err = _Py_INIT_NO_MEMORY();
         return NULL;
     }
 
@@ -538,7 +545,7 @@ pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str)
     wchar_t **options2 = (wchar_t **)PyMem_RawRealloc(list->options, size);
     if (options2 == NULL) {
         PyMem_RawFree(str2);
-        pymain->err = INIT_NO_MEMORY();
+        pymain->err = _Py_INIT_NO_MEMORY();
         return -1;
     }
     options2[list->len] = str2;
@@ -571,7 +578,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain)
             size_t len = wcslen(_PyOS_optarg) + 1 + 1;
             wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len);
             if (command == NULL) {
-                pymain->err = INIT_NO_MEMORY();
+                pymain->err = _Py_INIT_NO_MEMORY();
                 return -1;
             }
             memcpy(command, _PyOS_optarg, len * sizeof(wchar_t));
@@ -717,7 +724,7 @@ pymain_add_xoptions(_PyMain *pymain)
     for (size_t i=0; i < options->len; i++) {
         wchar_t *option = options->options[i];
         if (_PySys_AddXOptionWithError(option) < 0) {
-            pymain->err = INIT_NO_MEMORY();
+            pymain->err = _Py_INIT_NO_MEMORY();
             return -1;
         }
     }
@@ -748,11 +755,11 @@ pymain_add_warnings_options(_PyMain *pymain)
     PySys_ResetWarnOptions();
 
     if (pymain_add_warnings_optlist(&pymain->env_warning_options) < 0) {
-        pymain->err = INIT_NO_MEMORY();
+        pymain->err = _Py_INIT_NO_MEMORY();
         return -1;
     }
     if (pymain_add_warnings_optlist(&pymain->cmdline.warning_options) < 0) {
-        pymain->err = INIT_NO_MEMORY();
+        pymain->err = _Py_INIT_NO_MEMORY();
         return -1;
     }
     return 0;
@@ -801,7 +808,7 @@ pymain_warnings_envvar(_PyMain *pymain)
            C89 wcstok */
         buf = (char *)PyMem_RawMalloc(strlen(p) + 1);
         if (buf == NULL) {
-            pymain->err = INIT_NO_MEMORY();
+            pymain->err = _Py_INIT_NO_MEMORY();
             return -1;
         }
         strcpy(buf, p);
@@ -811,13 +818,7 @@ pymain_warnings_envvar(_PyMain *pymain)
             size_t len;
             wchar_t *warning = Py_DecodeLocale(p, &len);
             if (warning == NULL) {
-                if (len == (size_t)-2) {
-                    pymain->err = _Py_INIT_ERR("failed to decode "
-                                               "PYTHONWARNINGS");
-                }
-                else {
-                    pymain->err = INIT_NO_MEMORY();
-                }
+                SET_DECODE_ERROR("PYTHONWARNINGS environment variable", len);
                 return -1;
             }
             if (pymain_optlist_append(pymain, &pymain->env_warning_options,
@@ -902,7 +903,7 @@ pymain_get_program_name(_PyMain *pymain)
 
         buffer = PyMem_RawMalloc(len * sizeof(wchar_t));
         if (buffer == NULL) {
-            pymain->err = INIT_NO_MEMORY();
+            pymain->err = _Py_INIT_NO_MEMORY();
             return -1;
         }
 
@@ -919,15 +920,8 @@ pymain_get_program_name(_PyMain *pymain)
             size_t len;
             wchar_t* wbuf = Py_DecodeLocale(pyvenv_launcher, &len);
             if (wbuf == NULL) {
-                if (len == (size_t)-2) {
-                    pymain->err = _Py_INIT_ERR("failed to decode "
-                                               "__PYVENV_LAUNCHER__");
-                    return -1;
-                }
-                else {
-                    pymain->err = INIT_NO_MEMORY();
-                    return -1;
-                }
+                SET_DECODE_ERROR("__PYVENV_LAUNCHER__", len);
+                return -1;
             }
             pymain->program_name = wbuf;
         }
@@ -1403,7 +1397,7 @@ pymain_get_env_var_dup(_PyMain *pymain, wchar_t **dest,
             return -2;
         }
         else {
-            pymain->err = INIT_NO_MEMORY();
+            pymain->err = _Py_INIT_NO_MEMORY();
             return -1;
         }
     }
@@ -1421,7 +1415,7 @@ pymain_init_pythonpath(_PyMain *pymain)
                                      L"PYTHONPATH", "PYTHONPATH");
     if (res < 0) {
         if (res == -2) {
-            pymain->err = _Py_INIT_ERR("failed to decode PYTHONPATH");
+            SET_DECODE_ERROR("PYTHONPATH", (size_t)-2);
         }
         return -1;
     }
@@ -1450,7 +1444,7 @@ pymain_init_pythonhome(_PyMain *pymain)
                                      L"PYTHONHOME", "PYTHONHOME");
     if (res < 0) {
         if (res == -2) {
-            pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME");
+            SET_DECODE_ERROR("PYTHONHOME", (size_t)-2);
         }
         return -1;
     }

Python/pylifecycle.c

@@ -1477,8 +1477,9 @@ Py_SetPythonHome(wchar_t *home)
     default_home = home;
 }
 
-wchar_t *
-_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config)
+
+_PyInitError
+_Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config, wchar_t **homep)
 {
     /* Use a static buffer to avoid heap memory allocation failure.
        Py_GetPythonHome() doesn't allow to report error, and the caller
@@ -1486,32 +1487,40 @@ _Py_GetPythonHomeWithConfig(const _PyMainInterpreterConfig *config)
     static wchar_t buffer[MAXPATHLEN+1];
 
     if (default_home) {
-        return default_home;
+        *homep = default_home;
+        return _Py_INIT_OK();
     }
 
     if (config) {
-        return config->pythonhome;
+        *homep = config->pythonhome;
+        return _Py_INIT_OK();
     }
 
     char *home = Py_GETENV("PYTHONHOME");
     if (!home) {
-        return NULL;
+        *homep = NULL;
+        return _Py_INIT_OK();
     }
 
     size_t size = Py_ARRAY_LENGTH(buffer);
     size_t r = mbstowcs(buffer, home, size);
     if (r == (size_t)-1 || r >= size) {
         /* conversion failed or the static buffer is too small */
-        return NULL;
+        *homep = NULL;
+        return _Py_INIT_ERR("failed to decode PYTHONHOME environment variable");
     }
 
-    return buffer;
+    *homep = buffer;
+    return _Py_INIT_OK();
 }
 
 wchar_t *
 Py_GetPythonHome(void)
 {
-    return _Py_GetPythonHomeWithConfig(NULL);
+    wchar_t *home;
+    /* Ignore error */
+    (void)_Py_GetPythonHomeWithConfig(NULL, &home);
+    return home;
 }
 
 /* Add the __main__ module */