bpo-32030: Add _PyCoreConfig.module_search_paths (#4954)

_PyCoreConfig_Read() is now responsible to compute sys.path.
So sys.path is now computed before calling _Py_InitializeCore().

Changes:

* Add module_search_path, module_search_paths, executable, prefix,
  base_prefix, exec_prefix and base_exec_prefix to _PyCoreConfig.
* _PyMainInterpreterConfig_Read() now only converts wchar_t** lists
  into a Python list, it doesn't compute sys.path anymore.

Include/pystate.h

@@ -26,29 +26,24 @@ typedef PyObject* (*_PyFrameEvalFunction)(struct _frame *, int);
 
 typedef struct {
     int install_signal_handlers;  /* Install signal handlers? -1 means unset */
-    int ignore_environment; /* -E */
+
+    int ignore_environment; /* -E, Py_IgnoreEnvironmentFlag */
     int use_hash_seed;      /* PYTHONHASHSEED=x */
     unsigned long hash_seed;
-    int _disable_importlib; /* Needed by freeze_importlib */
     const char *allocator;  /* Memory allocator: _PyMem_SetupAllocators() */
-    int dev_mode;           /* -X dev */
+    int dev_mode;           /* PYTHONDEVMODE, -X dev */
-    int faulthandler;       /* -X faulthandler */
+    int faulthandler;       /* PYTHONFAULTHANDLER, -X faulthandler */
-    int tracemalloc;        /* -X tracemalloc=N */
+    int tracemalloc;        /* PYTHONTRACEMALLOC, -X tracemalloc=N */
-    int import_time;        /* -X importtime */
+    int import_time;        /* PYTHONPROFILEIMPORTTIME, -X importtime */
     int show_ref_count;     /* -X showrefcount */
     int show_alloc_count;   /* -X showalloccount */
     int dump_refs;          /* PYTHONDUMPREFS */
     int malloc_stats;       /* PYTHONMALLOCSTATS */
     int coerce_c_locale;    /* PYTHONCOERCECLOCALE, -1 means unknown */
     int coerce_c_locale_warn; /* PYTHONCOERCECLOCALE=warn */
-    int utf8_mode;          /* -X utf8 or PYTHONUTF8 environment variable,
+    int utf8_mode;          /* PYTHONUTF8, -X utf8; -1 means unknown */
-                               -1 means unknown */
 
-    wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
-    wchar_t *home;          /* PYTHONHOME environment variable,
-                               see also Py_SetPythonHome(). */
     wchar_t *program_name;  /* Program name, see also Py_GetProgramName() */
-
     int argc;               /* Number of command line arguments,
                                -1 means unset */
     wchar_t **argv;         /* Command line arguments */
@@ -59,6 +54,24 @@ typedef struct {
 
     int nwarnoption;        /* Number of warnings options */
     wchar_t **warnoptions;  /* Warnings options */
+
+    /* Path configuration inputs */
+    wchar_t *module_search_path_env; /* PYTHONPATH environment variable */
+    wchar_t *home;          /* PYTHONHOME environment variable,
+                               see also Py_SetPythonHome(). */
+
+    /* Path configuration outputs */
+    int nmodule_search_path;        /* Number of sys.path paths,
+                                       -1 means unset */
+    wchar_t **module_search_paths;  /* sys.path paths */
+    wchar_t *executable;    /* sys.executable */
+    wchar_t *prefix;        /* sys.prefix */
+    wchar_t *base_prefix;   /* sys.base_prefix */
+    wchar_t *exec_prefix;   /* sys.exec_prefix */
+    wchar_t *base_exec_prefix;  /* sys.base_exec_prefix */
+
+    /* Private fields */
+    int _disable_importlib; /* Needed by freeze_importlib */
 } _PyCoreConfig;
 
 #define _PyCoreConfig_INIT \
@@ -67,7 +80,8 @@ typedef struct {
         .use_hash_seed = -1, \
         .coerce_c_locale = -1, \
         .utf8_mode = -1, \
-        .argc = -1}
+        .argc = -1, \
+        .nmodule_search_path = -1}
 /* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */
 
 /* Placeholders while working on the new configuration API

Modules/main.c

@@ -1048,36 +1048,6 @@ config_init_warnoptions(_PyCoreConfig *config, _Py_CommandLineDetails *cmdline)
 }
 
 
-static int
-main_config_init_warnoptions(_PyMainInterpreterConfig *main_config,
-                             const _PyCoreConfig *config)
-{
-    PyObject *warnoptions = PyList_New(0);
-    if (warnoptions == NULL) {
-        return -1;
-    }
-
-    for (int i = 0; i < config->nwarnoption; i++) {
-        PyObject *option = PyUnicode_FromWideChar(config->warnoptions[i], -1);
-        if (option == NULL) {
-            goto error;
-        }
-        if (PyList_Append(warnoptions, option)) {
-            Py_DECREF(option);
-            goto error;
-        }
-        Py_DECREF(option);
-    }
-
-    main_config->warnoptions = warnoptions;
-    return 0;
-
-error:
-    Py_DECREF(warnoptions);
-    return -1;
-}
-
-
 /* Get warning options from PYTHONWARNINGS environment variable.
    Return 0 on success.
    Set pymain->err and return -1 on error. */
@@ -1324,34 +1294,25 @@ pymain_init_core_argv(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
 }
 
 
-static int
+static PyObject*
-main_config_init_argv(_PyMainInterpreterConfig *main_config,
+wstrlist_as_pylist(int len, wchar_t **list)
-                      const _PyCoreConfig *config)
 {
-    if (config->argc < 0) {
+    assert(list != NULL || len < 1);
-        return 0;
+
+    PyObject *pylist = PyList_New(len);
+    if (pylist == NULL) {
+        return NULL;
     }
 
-    int argc = config->argc;
+    for (int i = 0; i < len; i++) {
-    wchar_t** argv = config->argv;
+        PyObject *v = PyUnicode_FromWideChar(list[i], -1);
-    assert(argc >= 1 && argv != NULL);
-
-    PyObject *list = PyList_New(argc);
-    if (list == NULL) {
-        return -1;
-    }
-
-    for (int i = 0; i < argc; i++) {
-        PyObject *v = PyUnicode_FromWideChar(argv[i], -1);
         if (v == NULL) {
-            Py_DECREF(list);
+            Py_DECREF(pylist);
-            return -1;
+            return NULL;
         }
-        PyList_SET_ITEM(list, i, v);
+        PyList_SET_ITEM(pylist, i, v);
     }
-
+    return pylist;
-    main_config->argv = list;
-    return 0;
 }
 
 
@@ -1948,11 +1909,22 @@ pymain_read_conf_impl(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
         return -1;
     }
 
+    /* Global configuration variables should be set to read the core
+       configuration, and then get again to get updated values.
+
+       _PyPathConfig_Init() tests !Py_FrozenFlag to avoid some warnings.
+       Moreover, on Windows, it modifies Py_IsolatedFlag and Py_NoSiteFlag
+       variables if a "._pth" file is found. */
+    pymain_set_global_config(pymain, cmdline);
+
     err = _PyCoreConfig_Read(config);
     if (_Py_INIT_FAILED(err)) {
         pymain->err = err;
         return -1;
     }
+
+    Py_UTF8Mode = pymain->config.utf8_mode;
+    pymain_get_global_config(pymain, cmdline);
     return 0;
 }
 
@@ -1976,6 +1948,8 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
     int locale_coerced = 0;
     int loops = 0;
     int init_ignore_env = pymain->config.ignore_environment;
+    int init_isolated = cmdline->isolated;
+    int init_no_site = cmdline->no_site_import;
 
     while (1) {
         int utf8_mode = pymain->config.utf8_mode;
@@ -2033,9 +2007,12 @@ pymain_read_conf(_PyMain *pymain, _Py_CommandLineDetails *cmdline)
 
         /* Reset the configuration, except UTF-8 Mode. Set Py_UTF8Mode for
            Py_DecodeLocale(). Reset Py_IgnoreEnvironmentFlag, modified by
-           pymain_read_conf_impl(). */
+           pymain_read_conf_impl(). Reset Py_IsolatedFlag and Py_NoSiteFlag
+           modified by _PyCoreConfig_Read(). */
         Py_UTF8Mode = pymain->config.utf8_mode;
         Py_IgnoreEnvironmentFlag = init_ignore_env;
+        Py_IsolatedFlag = init_isolated;
+        Py_NoSiteFlag = init_no_site;
         _PyCoreConfig_Clear(&pymain->config);
         pymain_clear_cmdline(pymain, cmdline);
         pymain_get_global_config(pymain, cmdline);
@@ -2083,6 +2060,101 @@ config_init_locale(_PyCoreConfig *config)
 }
 
 
+static _PyInitError
+config_init_module_search_paths(_PyCoreConfig *config)
+{
+    assert(config->module_search_paths == NULL);
+    assert(config->nmodule_search_path < 0);
+
+    config->nmodule_search_path = 0;
+
+    const wchar_t *sys_path = Py_GetPath();
+    const wchar_t delim = DELIM;
+    const wchar_t *p = sys_path;
+    while (1) {
+        p = wcschr(sys_path, delim);
+        if (p == NULL) {
+            p = sys_path + wcslen(sys_path); /* End of string */
+        }
+
+        size_t path_len = (p - sys_path);
+        wchar_t *path = PyMem_RawMalloc((path_len + 1) * sizeof(wchar_t));
+        if (path == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+        memcpy(path, sys_path, path_len * sizeof(wchar_t));
+        path[path_len] = L'\0';
+
+        _PyInitError err = wstrlist_append(&config->nmodule_search_path,
+                                           &config->module_search_paths,
+                                           path);
+        PyMem_RawFree(path);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
+
+        if (*p == '\0') {
+            break;
+        }
+        sys_path = p + 1;
+    }
+    return _Py_INIT_OK();
+}
+
+
+static _PyInitError
+config_init_path_config(_PyCoreConfig *config)
+{
+    _PyInitError err = _PyPathConfig_Init(config);
+    if (_Py_INIT_FAILED(err)) {
+        return err;
+    }
+
+    if (config->nmodule_search_path < 0) {
+        err = config_init_module_search_paths(config);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
+    }
+
+    if (config->executable == NULL) {
+        config->executable = _PyMem_RawWcsdup(Py_GetProgramFullPath());
+        if (config->executable == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    if (config->prefix == NULL) {
+        config->prefix = _PyMem_RawWcsdup(Py_GetPrefix());
+        if (config->prefix == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    if (config->exec_prefix == NULL) {
+        config->exec_prefix = _PyMem_RawWcsdup(Py_GetExecPrefix());
+        if (config->exec_prefix == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    if (config->base_prefix == NULL) {
+        config->base_prefix = _PyMem_RawWcsdup(config->prefix);
+        if (config->base_prefix == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    if (config->base_exec_prefix == NULL) {
+        config->base_exec_prefix = _PyMem_RawWcsdup(config->exec_prefix);
+        if (config->base_exec_prefix == NULL) {
+            return _Py_INIT_NO_MEMORY();
+        }
+    }
+
+    return _Py_INIT_OK();
+}
+
 /* Read configuration settings from standard locations
  *
  * This function doesn't make any changes to the interpreter state - it
@@ -2140,6 +2212,12 @@ _PyCoreConfig_Read(_PyCoreConfig *config)
         config->install_signal_handlers = 1;
     }
 
+    if (!config->_disable_importlib) {
+        err = config_init_path_config(config);
+        if (_Py_INIT_FAILED(err)) {
+            return err;
+        }
+    }
     return _Py_INIT_OK();
 }
 
@@ -2152,26 +2230,33 @@ _PyCoreConfig_Clear(_PyCoreConfig *config)
         PyMem_RawFree(ATTR); \
         ATTR = NULL; \
     } while (0)
+#define CLEAR_WSTRLIST(LEN, LIST) \
+    do { \
+        clear_wstrlist(LEN, LIST); \
+        LEN = 0; \
+        LIST = NULL; \
+    } while (0)
 
     CLEAR(config->module_search_path_env);
     CLEAR(config->home);
     CLEAR(config->program_name);
     CLEAR(config->program);
 
-    if (config->argc >= 0) {
+    CLEAR_WSTRLIST(config->argc, config->argv);
-        clear_wstrlist(config->argc, config->argv);
+    config->argc = -1;
-        config->argc = -1;
-        config->argv = NULL;
-    }
 
-    clear_wstrlist(config->nwarnoption, config->warnoptions);
+    CLEAR_WSTRLIST(config->nwarnoption, config->warnoptions);
-    config->nwarnoption = 0;
+    CLEAR_WSTRLIST(config->nxoption, config->xoptions);
-    config->warnoptions = NULL;
+    CLEAR_WSTRLIST(config->nmodule_search_path, config->module_search_paths);
+    config->nmodule_search_path = -1;
 
-    clear_wstrlist(config->nxoption, config->xoptions);
+    CLEAR(config->executable);
-    config->nxoption = 0;
+    CLEAR(config->prefix);
-    config->xoptions = NULL;
+    CLEAR(config->base_prefix);
+    CLEAR(config->exec_prefix);
+    CLEAR(config->base_exec_prefix);
 #undef CLEAR
+#undef CLEAR_WSTRLIST
 }
 
 
@@ -2190,6 +2275,16 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
             } \
         } \
     } while (0)
+#define COPY_WSTRLIST(LEN, LIST) \
+    do { \
+        if (config2->LIST != NULL) { \
+            config->LIST = copy_wstrlist(config2->LEN, config2->LIST); \
+            if (config->LIST == NULL) { \
+                return -1; \
+            } \
+        } \
+        config->LEN = config2->LEN; \
+    } while (0)
 
     COPY_ATTR(ignore_environment);
     COPY_ATTR(use_hash_seed);
@@ -2211,33 +2306,20 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2)
     COPY_STR_ATTR(program_name);
     COPY_STR_ATTR(program);
 
-    if (config2->argc >= 0) {
+    COPY_WSTRLIST(argc, argv);
-        wchar_t **argv = copy_wstrlist(config2->argc, config2->argv);
+    COPY_WSTRLIST(nwarnoption, warnoptions);
-        if (argv == NULL) {
+    COPY_WSTRLIST(nxoption, xoptions);
-            return -1;
+    COPY_WSTRLIST(nmodule_search_path, module_search_paths);
-        }
-        config->argv = argv;
-    }
-    COPY_ATTR(argc);
 
-    if (config2->nwarnoption > 0) {
+    COPY_STR_ATTR(executable);
-        config->warnoptions = copy_wstrlist(config2->nwarnoption, config2->warnoptions);
+    COPY_STR_ATTR(prefix);
-        if (config->warnoptions == NULL) {
+    COPY_STR_ATTR(base_prefix);
-            return -1;
+    COPY_STR_ATTR(exec_prefix);
-        }
+    COPY_STR_ATTR(base_exec_prefix);
-    }
-    COPY_ATTR(nwarnoption);
-
-    if (config2->nxoption > 0) {
-        config->xoptions = copy_wstrlist(config2->nxoption, config2->xoptions);
-        if (config->xoptions == NULL) {
-            return -1;
-        }
-    }
-    COPY_ATTR(nxoption);
 
 #undef COPY_ATTR
 #undef COPY_STR_ATTR
+#undef COPY_WSTRLIST
     return 0;
 }
 
@@ -2313,52 +2395,10 @@ _PyMainInterpreterConfig_Copy(_PyMainInterpreterConfig *config,
 
 
 
-static PyObject *
-create_path_list(const wchar_t *path, wchar_t delim)
-{
-    int i, n;
-    const wchar_t *p;
-    PyObject *v;
-
-    n = 1;
-    p = path;
-    while ((p = wcschr(p, delim)) != NULL) {
-        n++;
-        p++;
-    }
-    v = PyList_New(n);
-    if (v == NULL) {
-        return NULL;
-    }
-    for (i = 0; ; i++) {
-        p = wcschr(path, delim);
-        if (p == NULL) {
-            p = path + wcslen(path); /* End of string */
-        }
-        PyObject *w = PyUnicode_FromWideChar(path, (Py_ssize_t)(p - path));
-        if (w == NULL) {
-            Py_DECREF(v);
-            return NULL;
-        }
-        PyList_SET_ITEM(v, i, w);
-        if (*p == '\0') {
-            break;
-        }
-        path = p+1;
-    }
-    return v;
-}
-
-
 _PyInitError
 _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config,
                               const _PyCoreConfig *config)
 {
-    _PyInitError err = _PyPathConfig_Init(config);
-    if (_Py_INIT_FAILED(err)) {
-        return err;
-    }
-
     if (main_config->install_signal_handlers < 0) {
         main_config->install_signal_handlers = config->install_signal_handlers;
     }
@@ -2370,59 +2410,46 @@ _PyMainInterpreterConfig_Read(_PyMainInterpreterConfig *main_config,
         }
     }
 
-    if (main_config->argv == NULL) {
+#define COPY_WSTR(ATTR) \
-        if (main_config_init_argv(main_config, config) < 0) {
+    do { \
-            return _Py_INIT_ERR("failed to create sys.argv");
+        if (main_config->ATTR == NULL) { \
-        }
+            main_config->ATTR = PyUnicode_FromWideChar(config->ATTR, -1); \
+            if (main_config->ATTR == NULL) { \
+                return _Py_INIT_NO_MEMORY(); \
+            } \
+        } \
+    } while (0)
+#define COPY_WSTRLIST(ATTR, LEN, LIST) \
+    do { \
+        if (ATTR == NULL) { \
+            ATTR = wstrlist_as_pylist(LEN, LIST); \
+            if (ATTR == NULL) { \
+                return _Py_INIT_NO_MEMORY(); \
+            } \
+        } \
+    } while (0)
+
+    COPY_WSTRLIST(main_config->warnoptions,
+                  config->nwarnoption, config->warnoptions);
+    if (config->argc >= 0) {
+        COPY_WSTRLIST(main_config->argv,
+                      config->argc, config->argv);
     }
 
-    if (main_config->warnoptions == NULL) {
+    if (!config->_disable_importlib) {
-        if (main_config_init_warnoptions(main_config, config) < 0) {
+        COPY_WSTR(executable);
-            return _Py_INIT_NO_MEMORY();
+        COPY_WSTR(prefix);
-        }
+        COPY_WSTR(base_prefix);
+        COPY_WSTR(exec_prefix);
+        COPY_WSTR(base_exec_prefix);
+
+        COPY_WSTRLIST(main_config->module_search_path,
+                      config->nmodule_search_path, config->module_search_paths);
     }
 
-    if (main_config->module_search_path == NULL &&
-        !config->_disable_importlib)
-    {
-        wchar_t *sys_path = Py_GetPath();
-        main_config->module_search_path = create_path_list(sys_path, DELIM);
-        if (main_config->module_search_path == NULL) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-
-    if (main_config->executable == NULL) {
-        main_config->executable = PyUnicode_FromWideChar(Py_GetProgramFullPath(), -1);
-        if (main_config->executable == NULL) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-
-    if (main_config->prefix == NULL) {
-        main_config->prefix = PyUnicode_FromWideChar(Py_GetPrefix(), -1);
-        if (main_config->prefix == NULL) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-
-    if (main_config->exec_prefix == NULL) {
-        main_config->exec_prefix = PyUnicode_FromWideChar(Py_GetExecPrefix(), -1);
-        if (main_config->exec_prefix == NULL) {
-            return _Py_INIT_NO_MEMORY();
-        }
-    }
-
-    if (main_config->base_prefix == NULL) {
-        Py_INCREF(main_config->prefix);
-        main_config->base_prefix = main_config->prefix;
-    }
-
-    if (main_config->base_exec_prefix == NULL) {
-        Py_INCREF(main_config->exec_prefix);
-        main_config->base_exec_prefix = main_config->exec_prefix;
-    }
     return _Py_INIT_OK();
+#undef COPY_WSTR
+#undef COPY_WSTRLIST
 }
 
 

PC/getpathp.c

@@ -692,6 +692,9 @@ calculate_pth_file(_PyPathConfig *config, wchar_t *prefix)
         return 0;
     }
 
+    /* FIXME, bpo-32030: Global configuration variables should not be modified
+       here, _PyPathConfig_Init() is called early in Python initialization:
+       see pymain_cmdline(). */
     return read_pth_file(config, prefix, spbuffer,
                          &Py_IsolatedFlag, &Py_NoSiteFlag);
 }