bpo-32030: Add _PyCoreConfig.module_search_path_env (#4504)

Changes:

* Py_Main() initializes _PyCoreConfig.module_search_path_env from
  the PYTHONPATH environment variable.
* PyInterpreterState_New() now initializes core_config and config
  fields
* Compute sys.path a little bit ealier in
  _Py_InitializeMainInterpreter() and new_interpreter()
* Add _Py_GetPathWithConfig() private function.

Include/pylifecycle.h

@@ -93,6 +93,9 @@ PyAPI_FUNC(wchar_t *) Py_GetProgramFullPath(void);
 PyAPI_FUNC(wchar_t *) Py_GetPrefix(void);
 PyAPI_FUNC(wchar_t *) Py_GetExecPrefix(void);
 PyAPI_FUNC(wchar_t *) Py_GetPath(void);
+#ifdef Py_BUILD_CORE
+PyAPI_FUNC(wchar_t *) _Py_GetPathWithConfig(_PyCoreConfig *config);
+#endif
 PyAPI_FUNC(void)      Py_SetPath(const wchar_t *);
 #ifdef MS_WINDOWS
 int _Py_CheckPython3();

Include/pystate.h

@@ -30,6 +30,7 @@ typedef struct {
     unsigned long hash_seed;
     int _disable_importlib; /* Needed by freeze_importlib */
     const char *allocator;  /* Memory allocator: _PyMem_SetupAllocators() */
+    wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
     int dev_mode;           /* -X dev */
     int faulthandler;       /* -X faulthandler */
     int tracemalloc;        /* -X tracemalloc=N */
@@ -39,11 +40,13 @@ typedef struct {
 } _PyCoreConfig;
 
 #define _PyCoreConfig_INIT \
-    {.ignore_environment = 0, \
+    (_PyCoreConfig){\
+     .ignore_environment = 0, \
      .use_hash_seed = -1, \
      .hash_seed = 0, \
      ._disable_importlib = 0, \
      .allocator = NULL, \
+     .module_search_path_env = NULL, \
      .dev_mode = 0, \
      .faulthandler = 0, \
      .tracemalloc = 0, \
@@ -61,7 +64,9 @@ typedef struct {
     int install_signal_handlers;
 } _PyMainInterpreterConfig;
 
-#define _PyMainInterpreterConfig_INIT {-1}
+#define _PyMainInterpreterConfig_INIT \
+    (_PyMainInterpreterConfig){\
+     .install_signal_handlers = -1}
 
 typedef struct _is {
 

Modules/getpath.c

@@ -456,14 +456,12 @@ search_for_exec_prefix(wchar_t *argv0_path, wchar_t *home,
 }
 
 static void
-calculate_path(void)
+calculate_path(_PyCoreConfig *core_config)
 {
     extern wchar_t *Py_GetProgramName(void);
 
     static const wchar_t delimiter[2] = {DELIM, '\0'};
     static const wchar_t separator[2] = {SEP, '\0'};
-    char *_rtpypath = Py_GETENV("PYTHONPATH"); /* XXX use wide version on Windows */
-    wchar_t *rtpypath = NULL;
     wchar_t *home = Py_GetPythonHome();
     char *_path = getenv("PATH");
     wchar_t *path_buffer = NULL;
@@ -707,11 +705,22 @@ calculate_path(void)
      */
     bufsz = 0;
 
-    if (_rtpypath && _rtpypath[0] != '\0') {
+    wchar_t *env_path = NULL;
-        size_t rtpypath_len;
+    if (core_config) {
-        rtpypath = Py_DecodeLocale(_rtpypath, &rtpypath_len);
+        if (core_config->module_search_path_env) {
-        if (rtpypath != NULL)
+            bufsz += wcslen(core_config->module_search_path_env) + 1;
-            bufsz += rtpypath_len + 1;
+        }
+    }
+    else {
+        char *env_pathb = Py_GETENV("PYTHONPATH");
+        if (env_pathb && env_pathb[0] != '\0') {
+            size_t env_path_len;
+            env_path = Py_DecodeLocale(env_pathb, &env_path_len);
+            /* FIXME: handle decoding and memory error */
+            if (env_path != NULL) {
+                bufsz += env_path_len + 1;
+            }
+        }
     }
 
     defpath = _pythonpath;
@@ -742,12 +751,20 @@ calculate_path(void)
     }
 
     /* Run-time value of $PYTHONPATH goes first */
-    if (rtpypath) {
+    buf[0] = '\0';
-        wcscpy(buf, rtpypath);
+    if (core_config) {
-        wcscat(buf, delimiter);
+        if (core_config->module_search_path_env) {
+            wcscpy(buf, core_config->module_search_path_env);
+            wcscat(buf, delimiter);
+        }
     }
-    else
+    else {
-        buf[0] = '\0';
+        if (env_path) {
+            wcscpy(buf, env_path);
+            wcscat(buf, delimiter);
+        }
+    }
+    PyMem_RawFree(env_path);
 
     /* Next is the default zip path */
     wcscat(buf, zip_path);
@@ -818,7 +835,6 @@ calculate_path(void)
     PyMem_RawFree(_prefix);
     PyMem_RawFree(_exec_prefix);
     PyMem_RawFree(lib_python);
-    PyMem_RawFree(rtpypath);
 }
 
 
@@ -841,11 +857,20 @@ Py_SetPath(const wchar_t *path)
     }
 }
 
+wchar_t *
+_Py_GetPathWithConfig(_PyCoreConfig *core_config)
+{
+    if (!module_search_path) {
+        calculate_path(core_config);
+    }
+    return module_search_path;
+}
+
 wchar_t *
 Py_GetPath(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return module_search_path;
 }
 
@@ -853,7 +878,7 @@ wchar_t *
 Py_GetPrefix(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return prefix;
 }
 
@@ -861,7 +886,7 @@ wchar_t *
 Py_GetExecPrefix(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return exec_prefix;
 }
 
@@ -869,7 +894,7 @@ wchar_t *
 Py_GetProgramFullPath(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return progpath;
 }
 

Modules/main.c

@@ -399,6 +399,7 @@ typedef struct {
     _PyInitError err;
     /* PYTHONWARNINGS env var */
     _Py_OptList env_warning_options;
+    /* PYTHONPATH env var */
     int argc;
     wchar_t **argv;
 } _PyMain;
@@ -441,6 +442,8 @@ pymain_free_impl(_PyMain *pymain)
     Py_CLEAR(pymain->main_importer_path);
     PyMem_RawFree(pymain->program_name);
 
+    PyMem_RawFree(pymain->core_config.module_search_path_env);
+
 #ifdef __INSURE__
     /* Insure++ is a memory analysis tool that aids in discovering
      * memory leaks and other memory problems.  On Python exit, the
@@ -502,15 +505,22 @@ error:
 
 
 static wchar_t*
-pymain_strdup(_PyMain *pymain, wchar_t *str)
+pymain_wstrdup(_PyMain *pymain, wchar_t *str)
 {
     size_t len = wcslen(str) + 1;  /* +1 for NUL character */
-    wchar_t *str2 = PyMem_RawMalloc(sizeof(wchar_t) * len);
+    if (len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) {
+        pymain->err = INIT_NO_MEMORY();
+        return NULL;
+    }
+
+    size_t size = len * sizeof(wchar_t);
+    wchar_t *str2 = PyMem_RawMalloc(size);
     if (str2 == NULL) {
         pymain->err = INIT_NO_MEMORY();
         return NULL;
     }
-    memcpy(str2, str, len * sizeof(wchar_t));
+
+    memcpy(str2, str, size);
     return str2;
 }
 
@@ -518,7 +528,7 @@ pymain_strdup(_PyMain *pymain, wchar_t *str)
 static int
 pymain_optlist_append(_PyMain *pymain, _Py_OptList *list, wchar_t *str)
 {
-    wchar_t *str2 = pymain_strdup(pymain, str);
+    wchar_t *str2 = pymain_wstrdup(pymain, str);
     if (str2 == NULL) {
         return -1;
     }
@@ -762,14 +772,12 @@ pymain_warnings_envvar(_PyMain *pymain)
     wchar_t *wp;
 
     if ((wp = _wgetenv(L"PYTHONWARNINGS")) && *wp != L'\0') {
-        wchar_t *buf, *warning, *context = NULL;
+        wchar_t *warning, *context = NULL;
 
-        buf = (wchar_t *)PyMem_RawMalloc((wcslen(wp) + 1) * sizeof(wchar_t));
+        wchar_t *buf = pymain_wstrdup(pymain, wp);
         if (buf == NULL) {
-            pymain->err = INIT_NO_MEMORY();
             return -1;
         }
-        wcscpy(buf, wp);
         for (warning = wcstok_s(buf, L",", &context);
              warning != NULL;
              warning = wcstok_s(NULL, L",", &context)) {
@@ -805,12 +813,11 @@ pymain_warnings_envvar(_PyMain *pymain)
                 if (len == (size_t)-2) {
                     pymain->err = _Py_INIT_ERR("failed to decode "
                                                "PYTHONWARNINGS");
-                    return -1;
                 }
                 else {
                     pymain->err = INIT_NO_MEMORY();
-                    return -1;
                 }
+                return -1;
             }
             if (pymain_optlist_append(pymain, &pymain->env_warning_options,
                                       warning) < 0) {
@@ -929,7 +936,7 @@ pymain_get_program_name(_PyMain *pymain)
 
     if (pymain->program_name == NULL) {
         /* Use argv[0] by default */
-        pymain->program_name = pymain_strdup(pymain, pymain->argv[0]);
+        pymain->program_name = pymain_wstrdup(pymain, pymain->argv[0]);
         if (pymain->program_name == NULL) {
             return -1;
         }
@@ -1362,6 +1369,48 @@ pymain_set_flags_from_env(_PyMain *pymain)
 }
 
 
+static int
+pymain_init_pythonpath(_PyMain *pymain)
+{
+    if (Py_IgnoreEnvironmentFlag) {
+        return 0;
+    }
+
+#ifdef MS_WINDOWS
+    wchar_t *path = _wgetenv(L"PYTHONPATH");
+    if (!path || path[0] == '\0') {
+        return 0;
+    }
+
+    wchar_t *path2 = pymain_wstrdup(pymain, path);
+    if (path2 == NULL) {
+        return -1;
+    }
+
+    pymain->core_config.module_search_path_env = path2;
+#else
+    char *path = pymain_get_env_var("PYTHONPATH");
+    if (!path) {
+        return 0;
+    }
+
+    size_t len;
+    wchar_t *wpath = Py_DecodeLocale(path, &len);
+    if (!wpath) {
+        if (len == (size_t)-2) {
+            pymain->err = _Py_INIT_ERR("failed to decode PYTHONHOME");
+        }
+        else {
+            pymain->err = INIT_NO_MEMORY();
+        }
+        return -1;
+    }
+    pymain->core_config.module_search_path_env = wpath;
+#endif
+    return 0;
+}
+
+
 static int
 pymain_parse_envvars(_PyMain *pymain)
 {
@@ -1383,6 +1432,9 @@ pymain_parse_envvars(_PyMain *pymain)
         return -1;
     }
     core_config->allocator = Py_GETENV("PYTHONMALLOC");
+    if (pymain_init_pythonpath(pymain) < 0) {
+        return -1;
+    }
 
     /* -X options */
     if (pymain_get_xoption(pymain, L"showrefcount")) {

PC/getpathp.c

@@ -624,7 +624,7 @@ error:
 
 
 static void
-calculate_path(void)
+calculate_path(_PyCoreConfig *core_config)
 {
     wchar_t argv0_path[MAXPATHLEN+1];
     wchar_t *buf;
@@ -637,8 +637,13 @@ calculate_path(void)
     wchar_t *userpath = NULL;
     wchar_t zip_path[MAXPATHLEN+1];
 
-    if (!Py_IgnoreEnvironmentFlag) {
+    if (core_config) {
+        envpath = core_config->module_search_path_env;
+    }
+    else if (!Py_IgnoreEnvironmentFlag) {
         envpath = _wgetenv(L"PYTHONPATH");
+        if (envpath && *envpath == '\0')
+            envpath = NULL;
     }
 
     get_progpath();
@@ -709,9 +714,6 @@ calculate_path(void)
     else
         wcscpy_s(prefix, MAXPATHLEN+1, pythonhome);
 
-    if (envpath && *envpath == '\0')
-        envpath = NULL;
-
 
     skiphome = pythonhome==NULL ? 0 : 1;
 #ifdef Py_ENABLE_SHARED
@@ -896,11 +898,20 @@ Py_SetPath(const wchar_t *path)
     }
 }
 
+wchar_t *
+_Py_GetPathWithConfig(_PyCoreConfig *core_config)
+{
+    if (!module_search_path) {
+        calculate_path(core_config);
+    }
+    return module_search_path;
+}
+
 wchar_t *
 Py_GetPath(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return module_search_path;
 }
 
@@ -908,7 +919,7 @@ wchar_t *
 Py_GetPrefix(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return prefix;
 }
 
@@ -922,7 +933,7 @@ wchar_t *
 Py_GetProgramFullPath(void)
 {
     if (!module_search_path)
-        calculate_path();
+        calculate_path(NULL);
     return progpath;
 }
 

Python/pylifecycle.c

@@ -48,8 +48,6 @@ _Py_IDENTIFIER(threading);
 extern "C" {
 #endif
 
-extern wchar_t *Py_GetPath(void);
-
 extern grammar _PyParser_Grammar; /* From graminit.c */
 
 /* Forward */
@@ -842,6 +840,11 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
     /* Now finish configuring the main interpreter */
     interp->config = *config;
 
+    /* GetPath may initialize state that _PySys_EndInit locks
+       in, and so has to be called first. */
+    /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */
+    wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config);
+
     if (interp->core_config._disable_importlib) {
         /* Special mode for freeze_importlib: run with no import system
          *
@@ -857,10 +860,7 @@ _Py_InitializeMainInterpreter(const _PyMainInterpreterConfig *config)
         return _Py_INIT_ERR("can't initialize time");
 
     /* Finish setting up the sys module and import system */
-    /* GetPath may initialize state that _PySys_EndInit locks
+    PySys_SetPath(sys_path);
-       in, and so has to be called first. */
-    /* TODO: Call Py_GetPath() in Py_ReadConfig, rather than here */
-    PySys_SetPath(Py_GetPath());
     if (_PySys_EndInit(interp->sysdict) < 0)
         return _Py_INIT_ERR("can't finish initializing sys");
 
@@ -1301,6 +1301,8 @@ new_interpreter(PyThreadState **tstate_p)
 
     /* XXX The following is lax in error checking */
 
+    wchar_t *sys_path = _Py_GetPathWithConfig(&interp->core_config);
+
     PyObject *modules = PyDict_New();
     if (modules == NULL) {
         return _Py_INIT_ERR("can't make modules dictionary");
@@ -1314,7 +1316,7 @@ new_interpreter(PyThreadState **tstate_p)
             goto handle_error;
         Py_INCREF(interp->sysdict);
         PyDict_SetItemString(interp->sysdict, "modules", modules);
-        PySys_SetPath(Py_GetPath());
+        PySys_SetPath(sys_path);
         _PySys_EndInit(interp->sysdict);
     }
 

Python/pystate.c

@@ -106,55 +106,60 @@ PyInterpreterState_New(void)
     PyInterpreterState *interp = (PyInterpreterState *)
                                  PyMem_RawMalloc(sizeof(PyInterpreterState));
 
-    if (interp != NULL) {
+    if (interp == NULL) {
-        interp->modules = NULL;
+        return NULL;
-        interp->modules_by_index = NULL;
+    }
-        interp->sysdict = NULL;
+
-        interp->builtins = NULL;
+
-        interp->builtins_copy = NULL;
+    interp->modules = NULL;
-        interp->tstate_head = NULL;
+    interp->modules_by_index = NULL;
-        interp->check_interval = 100;
+    interp->sysdict = NULL;
-        interp->num_threads = 0;
+    interp->builtins = NULL;
-        interp->pythread_stacksize = 0;
+    interp->builtins_copy = NULL;
-        interp->codec_search_path = NULL;
+    interp->tstate_head = NULL;
-        interp->codec_search_cache = NULL;
+    interp->check_interval = 100;
-        interp->codec_error_registry = NULL;
+    interp->num_threads = 0;
-        interp->codecs_initialized = 0;
+    interp->pythread_stacksize = 0;
-        interp->fscodec_initialized = 0;
+    interp->codec_search_path = NULL;
-        interp->importlib = NULL;
+    interp->codec_search_cache = NULL;
-        interp->import_func = NULL;
+    interp->codec_error_registry = NULL;
-        interp->eval_frame = _PyEval_EvalFrameDefault;
+    interp->codecs_initialized = 0;
-        interp->co_extra_user_count = 0;
+    interp->fscodec_initialized = 0;
+    interp->core_config = _PyCoreConfig_INIT;
+    interp->config = _PyMainInterpreterConfig_INIT;
+    interp->importlib = NULL;
+    interp->import_func = NULL;
+    interp->eval_frame = _PyEval_EvalFrameDefault;
+    interp->co_extra_user_count = 0;
 #ifdef HAVE_DLOPEN
 #if HAVE_DECL_RTLD_NOW
-        interp->dlopenflags = RTLD_NOW;
+    interp->dlopenflags = RTLD_NOW;
 #else
-        interp->dlopenflags = RTLD_LAZY;
+    interp->dlopenflags = RTLD_LAZY;
 #endif
 #endif
 #ifdef HAVE_FORK
-        interp->before_forkers = NULL;
+    interp->before_forkers = NULL;
-        interp->after_forkers_parent = NULL;
+    interp->after_forkers_parent = NULL;
-        interp->after_forkers_child = NULL;
+    interp->after_forkers_child = NULL;
 #endif
 
-        HEAD_LOCK();
+    HEAD_LOCK();
-        interp->next = _PyRuntime.interpreters.head;
+    interp->next = _PyRuntime.interpreters.head;
-        if (_PyRuntime.interpreters.main == NULL) {
+    if (_PyRuntime.interpreters.main == NULL) {
-            _PyRuntime.interpreters.main = interp;
+        _PyRuntime.interpreters.main = interp;
-        }
-        _PyRuntime.interpreters.head = interp;
-        if (_PyRuntime.interpreters.next_id < 0) {
-            /* overflow or Py_Initialize() not called! */
-            PyErr_SetString(PyExc_RuntimeError,
-                            "failed to get an interpreter ID");
-            interp = NULL;
-        } else {
-            interp->id = _PyRuntime.interpreters.next_id;
-            _PyRuntime.interpreters.next_id += 1;
-        }
-        HEAD_UNLOCK();
     }
+    _PyRuntime.interpreters.head = interp;
+    if (_PyRuntime.interpreters.next_id < 0) {
+        /* overflow or Py_Initialize() not called! */
+        PyErr_SetString(PyExc_RuntimeError,
+                        "failed to get an interpreter ID");
+        interp = NULL;
+    } else {
+        interp->id = _PyRuntime.interpreters.next_id;
+        _PyRuntime.interpreters.next_id += 1;
+    }
+    HEAD_UNLOCK();
 
     return interp;
 }