From 9e1b828265e6bfb58f1e0299bd78d8ff6347a2ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Nov 2020 13:21:52 +0100 Subject: [PATCH] bpo-42260: Compute the path config in the main init (GH-23211) The path configuration is now computed in the "main" initialization. The core initialization no longer computes it. * Add _PyConfig_Read() function to read the configuration without computing the path configuration. * pyinit_core() no longer computes the path configuration: it is now computed by init_interp_main(). * The path configuration output members of PyConfig are now optional: * executable * base_executable * prefix * base_prefix * exec_prefix * base_exec_prefix * _PySys_UpdateConfig() now skips NULL strings in PyConfig. * _testembed: Rename test_set_config() to test_init_set_config() for consistency with other tests. --- Include/internal/pycore_initconfig.h | 1 + Lib/test/_test_embed_set_config.py | 26 ++++++++---- Programs/_testembed.c | 4 +- Python/initconfig.c | 47 +++++++++------------- Python/pylifecycle.c | 59 +++++++++++++++------------- Python/sysmodule.c | 22 +++++++---- 6 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 325be5494d4..d8400b1c42e 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -152,6 +152,7 @@ extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); extern PyStatus _PyConfig_InitPathConfig(PyConfig *config); +extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config); extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); extern PyStatus _PyConfig_SetPyArgv( diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index a19f8db1584..82c5d829823 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -100,19 +100,19 @@ class SetConfigTests(unittest.TestCase): 'check_hash_pycs_mode', 'program_name', 'platlibdir', - 'executable', - 'base_executable', - 'prefix', - 'base_prefix', - 'exec_prefix', - 'base_exec_prefix', # optional wstr: # 'pythonpath_env' - # 'home', + # 'home' # 'pycache_prefix' # 'run_command' # 'run_module' # 'run_filename' + # 'executable' + # 'prefix' + # 'exec_prefix' + # 'base_executable' + # 'base_prefix' + # 'base_exec_prefix' ): value_tests.append((key, invalid_wstr)) type_tests.append((key, b'bytes')) @@ -217,6 +217,18 @@ class SetConfigTests(unittest.TestCase): self.set_config(base_executable="base_executable") self.assertEqual(sys._base_executable, "base_executable") + # When base_xxx is NULL, value is copied from xxxx + self.set_config( + executable='executable', + prefix="prefix", + exec_prefix="exec_prefix", + base_executable=None, + base_prefix=None, + base_exec_prefix=None) + self.assertEqual(sys._base_executable, "executable") + self.assertEqual(sys.base_prefix, "prefix") + self.assertEqual(sys.base_exec_prefix, "exec_prefix") + def test_path(self): self.set_config(module_search_paths_set=1, module_search_paths=['a', 'b', 'c']) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index cb3a23a101e..748ea8a8f33 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1547,7 +1547,7 @@ static int tune_config(void) } -static int test_set_config(void) +static int test_init_set_config(void) { // Initialize core PyConfig config; @@ -1742,7 +1742,7 @@ static struct TestCase TestCases[] = { {"test_init_setpath_config", test_init_setpath_config}, {"test_init_setpythonhome", test_init_setpythonhome}, {"test_init_warnoptions", test_init_warnoptions}, - {"test_init_set_config", test_set_config}, + {"test_init_set_config", test_init_set_config}, {"test_run_main", test_run_main}, {"test_get_argc_argv", test_get_argc_argv}, diff --git a/Python/initconfig.c b/Python/initconfig.c index e0811b56cb3..11db4a3ef59 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -619,15 +619,6 @@ config_check_consistency(const PyConfig *config) assert(_PyWideStringList_CheckConsistency(&config->warnoptions)); assert(_PyWideStringList_CheckConsistency(&config->module_search_paths)); assert(config->module_search_paths_set >= 0); - if (config->_install_importlib) { - /* don't check config->module_search_paths */ - assert(config->executable != NULL); - assert(config->base_executable != NULL); - assert(config->prefix != NULL); - assert(config->base_prefix != NULL); - assert(config->exec_prefix != NULL); - assert(config->base_exec_prefix != NULL); - } assert(config->platlibdir != NULL); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); @@ -1297,24 +1288,15 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_WSTR_OPT(home); GET_WSTR(platlibdir); + // Path configuration output GET_UINT(module_search_paths_set); GET_WSTRLIST(module_search_paths); - if (config->_install_importlib) { - GET_WSTR(executable); - GET_WSTR(base_executable); - GET_WSTR(prefix); - GET_WSTR(base_prefix); - GET_WSTR(exec_prefix); - GET_WSTR(base_exec_prefix); - } - else { - GET_WSTR_OPT(executable); - GET_WSTR_OPT(base_executable); - GET_WSTR_OPT(prefix); - GET_WSTR_OPT(base_prefix); - GET_WSTR_OPT(exec_prefix); - GET_WSTR_OPT(base_exec_prefix); - } + GET_WSTR_OPT(executable); + GET_WSTR_OPT(base_executable); + GET_WSTR_OPT(prefix); + GET_WSTR_OPT(base_prefix); + GET_WSTR_OPT(exec_prefix); + GET_WSTR_OPT(base_exec_prefix); GET_UINT(skip_source_first_line); GET_WSTR_OPT(run_command); @@ -2043,7 +2025,7 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) static PyStatus -config_read(PyConfig *config) +config_read(PyConfig *config, int compute_path_config) { PyStatus status; const PyPreConfig *preconfig = &_PyRuntime.preconfig; @@ -2087,7 +2069,7 @@ config_read(PyConfig *config) } } - if (config->_install_importlib) { + if (compute_path_config && config->_install_importlib) { status = _PyConfig_InitPathConfig(config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -2834,7 +2816,7 @@ PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, The only side effects are to modify config and to call _Py_SetArgcArgv(). */ PyStatus -PyConfig_Read(PyConfig *config) +_PyConfig_Read(PyConfig *config, int compute_path_config) { PyStatus status; @@ -2877,7 +2859,7 @@ PyConfig_Read(PyConfig *config) goto done; } - status = config_read(config); + status = config_read(config, compute_path_config); if (_PyStatus_EXCEPTION(status)) { goto done; } @@ -2892,6 +2874,13 @@ done: } +PyStatus +PyConfig_Read(PyConfig *config) +{ + return _PyConfig_Read(config, 1); +} + + PyObject* _Py_GetConfigsAsDict(void) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e34d6471e17..609e0a42e4d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -429,25 +429,20 @@ _Py_SetLocaleFromEnv(int category) static int -interpreter_set_config(const PyConfig *config) +interpreter_update_config(PyThreadState *tstate, int only_update_path_config) { - PyThreadState *tstate = PyThreadState_Get(); + const PyConfig *config = &tstate->interp->config; - PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - return -1; + if (!only_update_path_config) { + PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; + } } - status = _PyConfig_Copy(&tstate->interp->config, config); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - return -1; - } - config = &tstate->interp->config; - - if (config->_install_importlib && _Py_IsMainInterpreter(tstate)) { - status = _PyConfig_WritePathConfig(config); + if (_Py_IsMainInterpreter(tstate)) { + PyStatus status = _PyConfig_WritePathConfig(config); if (_PyStatus_EXCEPTION(status)) { _PyErr_SetFromPyStatus(status); return -1; @@ -465,6 +460,7 @@ interpreter_set_config(const PyConfig *config) int _PyInterpreterState_SetConfig(const PyConfig *src_config) { + PyThreadState *tstate = PyThreadState_Get(); int res = -1; PyConfig config; @@ -481,7 +477,13 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config) goto done; } - res = interpreter_set_config(&config); + status = _PyConfig_Copy(&tstate->interp->config, &config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + goto done; + } + + res = interpreter_update_config(tstate, 0); done: PyConfig_Clear(&config); @@ -763,13 +765,6 @@ pycore_init_import_warnings(PyThreadState *tstate, PyObject *sysmod) const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (config->_install_importlib) { - if (_Py_IsMainInterpreter(tstate)) { - status = _PyConfig_WritePathConfig(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - /* This call sets up builtin and frozen import support */ status = init_importlib(tstate, sysmod); if (_PyStatus_EXCEPTION(status)) { @@ -985,7 +980,9 @@ pyinit_core(_PyRuntimeState *runtime, goto done; } - status = PyConfig_Read(&config); + // Read the configuration, but don't compute the path configuration + // (it is computed in the main init). + status = _PyConfig_Read(&config, 0); if (_PyStatus_EXCEPTION(status)) { goto done; } @@ -1012,8 +1009,8 @@ done: static PyStatus pyinit_main_reconfigure(PyThreadState *tstate) { - if (_PySys_UpdateConfig(tstate) < 0) { - return _PyStatus_ERR("fail to update sys for the new conf"); + if (interpreter_update_config(tstate, 0) < 0) { + return _PyStatus_ERR("fail to reconfigure Python"); } return _PyStatus_OK(); } @@ -1041,14 +1038,20 @@ init_interp_main(PyThreadState *tstate) return _PyStatus_OK(); } + // Compute the path configuration + status = _PyConfig_InitPathConfig(&interp->config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + if (is_main_interp) { if (_PyTime_Init() < 0) { return _PyStatus_ERR("can't initialize time"); } } - if (_PySys_UpdateConfig(tstate) < 0) { - return _PyStatus_ERR("can't finish initializing sys"); + if (interpreter_update_config(tstate, 1) < 0) { + return _PyStatus_ERR("failed to update the Python config"); } status = init_importlib_external(tstate); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ae4f0eeb2ee..61741f7432d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2922,17 +2922,22 @@ _PySys_UpdateConfig(PyThreadState *tstate) #define SET_SYS_FROM_WSTR(KEY, VALUE) \ SET_SYS(KEY, PyUnicode_FromWideChar(VALUE, -1)); +#define COPY_WSTR(SYS_ATTR, WSTR) \ + if (WSTR != NULL) { \ + SET_SYS_FROM_WSTR(SYS_ATTR, WSTR); \ + } + if (config->module_search_paths_set) { COPY_LIST("path", config->module_search_paths); } - SET_SYS_FROM_WSTR("executable", config->executable); - SET_SYS_FROM_WSTR("_base_executable", config->base_executable); - SET_SYS_FROM_WSTR("prefix", config->prefix); - SET_SYS_FROM_WSTR("base_prefix", config->base_prefix); - SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix); - SET_SYS_FROM_WSTR("base_exec_prefix", config->base_exec_prefix); - SET_SYS_FROM_WSTR("platlibdir", config->platlibdir); + COPY_WSTR("executable", config->executable); + COPY_WSTR("_base_executable", config->base_executable); + COPY_WSTR("prefix", config->prefix); + COPY_WSTR("base_prefix", config->base_prefix); + COPY_WSTR("exec_prefix", config->exec_prefix); + COPY_WSTR("base_exec_prefix", config->base_exec_prefix); + COPY_WSTR("platlibdir", config->platlibdir); if (config->pycache_prefix != NULL) { SET_SYS_FROM_WSTR("pycache_prefix", config->pycache_prefix); @@ -2946,8 +2951,9 @@ _PySys_UpdateConfig(PyThreadState *tstate) SET_SYS("_xoptions", sys_create_xoptions_dict(config)); -#undef COPY_LIST #undef SET_SYS_FROM_WSTR +#undef COPY_LIST +#undef COPY_WSTR // sys.flags PyObject *flags = _PySys_GetObject(tstate, "flags"); // borrowed ref