bpo-42260: Compute the path config in the main init (GH-23211)

The path configuration is now computed in the "main" initialization.
The core initialization no longer computes it.

* Add _PyConfig_Read() function to read the configuration without
  computing the path configuration.
* pyinit_core() no longer computes the path configuration: it is now
  computed by init_interp_main().
* The path configuration output members of PyConfig are now optional:

  * executable
  * base_executable
  * prefix
  * base_prefix
  * exec_prefix
  * base_exec_prefix

* _PySys_UpdateConfig() now skips NULL strings in PyConfig.
* _testembed: Rename test_set_config() to test_init_set_config() for
  consistency with other tests.

Include/internal/pycore_initconfig.h

@@ -152,6 +152,7 @@ extern PyStatus _PyConfig_Copy(
     PyConfig *config,
     const PyConfig *config2);
 extern PyStatus _PyConfig_InitPathConfig(PyConfig *config);
+extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config);
 extern PyStatus _PyConfig_Write(const PyConfig *config,
     struct pyruntimestate *runtime);
 extern PyStatus _PyConfig_SetPyArgv(

Lib/test/_test_embed_set_config.py

@@ -100,19 +100,19 @@ class SetConfigTests(unittest.TestCase):
             'check_hash_pycs_mode',
             'program_name',
             'platlibdir',
-            'executable',
-            'base_executable',
-            'prefix',
-            'base_prefix',
-            'exec_prefix',
-            'base_exec_prefix',
             # optional wstr:
             # 'pythonpath_env'
-            # 'home',
+            # 'home'
             # 'pycache_prefix'
             # 'run_command'
             # 'run_module'
             # 'run_filename'
+            # 'executable'
+            # 'prefix'
+            # 'exec_prefix'
+            # 'base_executable'
+            # 'base_prefix'
+            # 'base_exec_prefix'
         ):
             value_tests.append((key, invalid_wstr))
             type_tests.append((key, b'bytes'))
@@ -217,6 +217,18 @@ class SetConfigTests(unittest.TestCase):
         self.set_config(base_executable="base_executable")
         self.assertEqual(sys._base_executable, "base_executable")
 
+        # When base_xxx is NULL, value is copied from xxxx
+        self.set_config(
+            executable='executable',
+            prefix="prefix",
+            exec_prefix="exec_prefix",
+            base_executable=None,
+            base_prefix=None,
+            base_exec_prefix=None)
+        self.assertEqual(sys._base_executable, "executable")
+        self.assertEqual(sys.base_prefix, "prefix")
+        self.assertEqual(sys.base_exec_prefix, "exec_prefix")
+
     def test_path(self):
         self.set_config(module_search_paths_set=1,
                         module_search_paths=['a', 'b', 'c'])

Programs/_testembed.c

@@ -1547,7 +1547,7 @@ static int tune_config(void)
 }
 
 
-static int test_set_config(void)
+static int test_init_set_config(void)
 {
     // Initialize core
     PyConfig config;
@@ -1742,7 +1742,7 @@ static struct TestCase TestCases[] = {
     {"test_init_setpath_config", test_init_setpath_config},
     {"test_init_setpythonhome", test_init_setpythonhome},
     {"test_init_warnoptions", test_init_warnoptions},
-    {"test_init_set_config", test_set_config},
+    {"test_init_set_config", test_init_set_config},
     {"test_run_main", test_run_main},
     {"test_get_argc_argv", test_get_argc_argv},
 

Python/initconfig.c

@@ -619,15 +619,6 @@ config_check_consistency(const PyConfig *config)
     assert(_PyWideStringList_CheckConsistency(&config->warnoptions));
     assert(_PyWideStringList_CheckConsistency(&config->module_search_paths));
     assert(config->module_search_paths_set >= 0);
-    if (config->_install_importlib) {
-        /* don't check config->module_search_paths */
-        assert(config->executable != NULL);
-        assert(config->base_executable != NULL);
-        assert(config->prefix != NULL);
-        assert(config->base_prefix != NULL);
-        assert(config->exec_prefix != NULL);
-        assert(config->base_exec_prefix != NULL);
-    }
     assert(config->platlibdir != NULL);
     assert(config->filesystem_encoding != NULL);
     assert(config->filesystem_errors != NULL);
@@ -1297,24 +1288,15 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_WSTR_OPT(home);
     GET_WSTR(platlibdir);
 
+    // Path configuration output
     GET_UINT(module_search_paths_set);
     GET_WSTRLIST(module_search_paths);
-    if (config->_install_importlib) {
-        GET_WSTR(executable);
-        GET_WSTR(base_executable);
-        GET_WSTR(prefix);
-        GET_WSTR(base_prefix);
-        GET_WSTR(exec_prefix);
-        GET_WSTR(base_exec_prefix);
-    }
-    else {
     GET_WSTR_OPT(executable);
     GET_WSTR_OPT(base_executable);
     GET_WSTR_OPT(prefix);
     GET_WSTR_OPT(base_prefix);
     GET_WSTR_OPT(exec_prefix);
     GET_WSTR_OPT(base_exec_prefix);
-    }
 
     GET_UINT(skip_source_first_line);
     GET_WSTR_OPT(run_command);
@@ -2043,7 +2025,7 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig)
 
 
 static PyStatus
-config_read(PyConfig *config)
+config_read(PyConfig *config, int compute_path_config)
 {
     PyStatus status;
     const PyPreConfig *preconfig = &_PyRuntime.preconfig;
@@ -2087,7 +2069,7 @@ config_read(PyConfig *config)
         }
     }
 
-    if (config->_install_importlib) {
+    if (compute_path_config && config->_install_importlib) {
         status = _PyConfig_InitPathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             return status;
@@ -2834,7 +2816,7 @@ PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list,
 
    The only side effects are to modify config and to call _Py_SetArgcArgv(). */
 PyStatus
-PyConfig_Read(PyConfig *config)
+_PyConfig_Read(PyConfig *config, int compute_path_config)
 {
     PyStatus status;
 
@@ -2877,7 +2859,7 @@ PyConfig_Read(PyConfig *config)
         goto done;
     }
 
-    status = config_read(config);
+    status = config_read(config, compute_path_config);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
@@ -2892,6 +2874,13 @@ done:
 }
 
 
+PyStatus
+PyConfig_Read(PyConfig *config)
+{
+    return _PyConfig_Read(config, 1);
+}
+
+
 PyObject*
 _Py_GetConfigsAsDict(void)
 {

Python/pylifecycle.c

@@ -429,25 +429,20 @@ _Py_SetLocaleFromEnv(int category)
 
 
 static int
-interpreter_set_config(const PyConfig *config)
+interpreter_update_config(PyThreadState *tstate, int only_update_path_config)
 {
-    PyThreadState *tstate = PyThreadState_Get();
+    const PyConfig *config = &tstate->interp->config;
 
+    if (!only_update_path_config) {
         PyStatus status = _PyConfig_Write(config, tstate->interp->runtime);
         if (_PyStatus_EXCEPTION(status)) {
             _PyErr_SetFromPyStatus(status);
             return -1;
         }
-
-    status = _PyConfig_Copy(&tstate->interp->config, config);
-    if (_PyStatus_EXCEPTION(status)) {
-        _PyErr_SetFromPyStatus(status);
-        return -1;
     }
-    config = &tstate->interp->config;
 
-    if (config->_install_importlib && _Py_IsMainInterpreter(tstate)) {
+    if (_Py_IsMainInterpreter(tstate)) {
-        status = _PyConfig_WritePathConfig(config);
+        PyStatus status = _PyConfig_WritePathConfig(config);
         if (_PyStatus_EXCEPTION(status)) {
             _PyErr_SetFromPyStatus(status);
             return -1;
@@ -465,6 +460,7 @@ interpreter_set_config(const PyConfig *config)
 int
 _PyInterpreterState_SetConfig(const PyConfig *src_config)
 {
+    PyThreadState *tstate = PyThreadState_Get();
     int res = -1;
 
     PyConfig config;
@@ -481,7 +477,13 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config)
         goto done;
     }
 
-    res = interpreter_set_config(&config);
+    status = _PyConfig_Copy(&tstate->interp->config, &config);
+    if (_PyStatus_EXCEPTION(status)) {
+        _PyErr_SetFromPyStatus(status);
+        goto done;
+    }
+
+    res = interpreter_update_config(tstate, 0);
 
 done:
     PyConfig_Clear(&config);
@@ -763,13 +765,6 @@ pycore_init_import_warnings(PyThreadState *tstate, PyObject *sysmod)
 
     const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp);
     if (config->_install_importlib) {
-        if (_Py_IsMainInterpreter(tstate)) {
-            status = _PyConfig_WritePathConfig(config);
-            if (_PyStatus_EXCEPTION(status)) {
-                return status;
-            }
-        }
-
         /* This call sets up builtin and frozen import support */
         status = init_importlib(tstate, sysmod);
         if (_PyStatus_EXCEPTION(status)) {
@@ -985,7 +980,9 @@ pyinit_core(_PyRuntimeState *runtime,
         goto done;
     }
 
-    status = PyConfig_Read(&config);
+    // Read the configuration, but don't compute the path configuration
+    // (it is computed in the main init).
+    status = _PyConfig_Read(&config, 0);
     if (_PyStatus_EXCEPTION(status)) {
         goto done;
     }
@@ -1012,8 +1009,8 @@ done:
 static PyStatus
 pyinit_main_reconfigure(PyThreadState *tstate)
 {
-    if (_PySys_UpdateConfig(tstate) < 0) {
+    if (interpreter_update_config(tstate, 0) < 0) {
-        return _PyStatus_ERR("fail to update sys for the new conf");
+        return _PyStatus_ERR("fail to reconfigure Python");
     }
     return _PyStatus_OK();
 }
@@ -1041,14 +1038,20 @@ init_interp_main(PyThreadState *tstate)
         return _PyStatus_OK();
     }
 
+    // Compute the path configuration
+    status = _PyConfig_InitPathConfig(&interp->config);
+    if (_PyStatus_EXCEPTION(status)) {
+        return status;
+    }
+
     if (is_main_interp) {
         if (_PyTime_Init() < 0) {
             return _PyStatus_ERR("can't initialize time");
         }
     }
 
-    if (_PySys_UpdateConfig(tstate) < 0) {
+    if (interpreter_update_config(tstate, 1) < 0) {
-        return _PyStatus_ERR("can't finish initializing sys");
+        return _PyStatus_ERR("failed to update the Python config");
     }
 
     status = init_importlib_external(tstate);

Python/sysmodule.c

@@ -2922,17 +2922,22 @@ _PySys_UpdateConfig(PyThreadState *tstate)
 #define SET_SYS_FROM_WSTR(KEY, VALUE) \
         SET_SYS(KEY, PyUnicode_FromWideChar(VALUE, -1));
 
+#define COPY_WSTR(SYS_ATTR, WSTR) \
+    if (WSTR != NULL) { \
+        SET_SYS_FROM_WSTR(SYS_ATTR, WSTR); \
+    }
+
     if (config->module_search_paths_set) {
         COPY_LIST("path", config->module_search_paths);
     }
 
-    SET_SYS_FROM_WSTR("executable", config->executable);
+    COPY_WSTR("executable", config->executable);
-    SET_SYS_FROM_WSTR("_base_executable", config->base_executable);
+    COPY_WSTR("_base_executable", config->base_executable);
-    SET_SYS_FROM_WSTR("prefix", config->prefix);
+    COPY_WSTR("prefix", config->prefix);
-    SET_SYS_FROM_WSTR("base_prefix", config->base_prefix);
+    COPY_WSTR("base_prefix", config->base_prefix);
-    SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix);
+    COPY_WSTR("exec_prefix", config->exec_prefix);
-    SET_SYS_FROM_WSTR("base_exec_prefix", config->base_exec_prefix);
+    COPY_WSTR("base_exec_prefix", config->base_exec_prefix);
-    SET_SYS_FROM_WSTR("platlibdir", config->platlibdir);
+    COPY_WSTR("platlibdir", config->platlibdir);
 
     if (config->pycache_prefix != NULL) {
         SET_SYS_FROM_WSTR("pycache_prefix", config->pycache_prefix);
@@ -2946,8 +2951,9 @@ _PySys_UpdateConfig(PyThreadState *tstate)
 
     SET_SYS("_xoptions", sys_create_xoptions_dict(config));
 
-#undef COPY_LIST
 #undef SET_SYS_FROM_WSTR
+#undef COPY_LIST
+#undef COPY_WSTR
 
     // sys.flags
     PyObject *flags = _PySys_GetObject(tstate, "flags");  // borrowed ref