From 9ef5dcaa0b3c7c7ba28dbb3ec0c9507d9d05e3a9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 16 May 2019 17:38:16 +0200 Subject: [PATCH] bpo-36763: Add _Py_InitializeMain() (GH-13362) * Add a private _Py_InitializeMain() function. * Add again _PyCoreConfig._init_main. * _Py_InitializeFromConfig() now uses _init_main to decide if _Py_InitializeMainInterpreter() should be called. * _PyCoreConfig: rename _frozen to pathconfig_warnings, its value is now the opposite of Py_FrozenFlag. * Add an unit test for _init_main=0 and _Py_InitializeMain(). --- Include/cpython/coreconfig.h | 13 ++++--- Include/cpython/pylifecycle.h | 1 + Lib/test/test_embed.py | 43 ++++++++++++++++------- Modules/getpath.c | 6 ++-- Programs/_freeze_importlib.c | 3 +- Programs/_testembed.c | 64 ++++++++++++++++++++++++++++------- Python/coreconfig.c | 12 ++++--- Python/frozenmain.c | 2 +- Python/pylifecycle.c | 17 +++++++++- Python/pythonrun.c | 9 +++++ 10 files changed, 130 insertions(+), 40 deletions(-) diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index a04342ea980..c2c55668406 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -398,10 +398,14 @@ typedef struct { See PEP 552 "Deterministic pycs" for more details. */ wchar_t *check_hash_pycs_mode; - /* If greater than 0, suppress _PyPathConfig_Calculate() warnings. + /* If greater than 0, suppress _PyPathConfig_Calculate() warnings on Unix. + The parameter has no effect on Windows. - If set to -1 (default), inherit Py_FrozenFlag value. */ - int _frozen; + If set to -1 (default), inherit !Py_FrozenFlag value. */ + int pathconfig_warnings; + + /* If equal to 0, stop Python initialization before the "main" phase */ + int _init_main; } _PyCoreConfig; @@ -438,7 +442,8 @@ typedef struct { .buffered_stdio = -1, \ ._install_importlib = 1, \ .check_hash_pycs_mode = NULL, \ - ._frozen = -1} + .pathconfig_warnings = -1, \ + ._init_main = 1} /* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ #ifdef __cplusplus diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 2366c774e7f..a3ab6c915ef 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -40,6 +40,7 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeFromWideArgs( const _PyCoreConfig *config, int argc, wchar_t **argv); +PyAPI_FUNC(_PyInitError) _Py_InitializeMain(void); PyAPI_FUNC(int) _Py_RunMain(void); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 3fabe5f9b25..c3c1a3e3bac 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -343,7 +343,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_install_importlib': 1, 'check_hash_pycs_mode': 'default', - '_frozen': 0, + 'pathconfig_warnings': 1, + '_init_main': 1, } if MS_WINDOWS: DEFAULT_PRE_CONFIG.update({ @@ -371,7 +372,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ('Py_DontWriteBytecodeFlag', 'write_bytecode', True), ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), - ('Py_FrozenFlag', '_frozen'), + ('Py_FrozenFlag', 'pathconfig_warnings', True), ('Py_IgnoreEnvironmentFlag', 'use_environment', True), ('Py_InspectFlag', 'inspect'), ('Py_InteractiveFlag', 'interactive'), @@ -500,7 +501,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(config['global_config'], expected) - def check_config(self, testname, expected_config, expected_preconfig, add_path=None): + def check_config(self, testname, expected_config, expected_preconfig, + add_path=None, stderr=None): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment for key in list(env): @@ -511,19 +513,22 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): env['PYTHONCOERCECLOCALE'] = '0' env['PYTHONUTF8'] = '0' - out, err = self.run_embedded_interpreter(testname, env=env) - # Ignore err - try: - config = json.loads(out) - except json.JSONDecodeError: - self.fail(f"fail to decode stdout: {out!r}") - expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig) expected_config = self.get_expected_config(expected_config, env, add_path) for key in self.COPY_PRE_CONFIG: if key not in expected_preconfig: expected_preconfig[key] = expected_config[key] + out, err = self.run_embedded_interpreter(testname, env=env) + if stderr is None and not expected_config['verbose']: + stderr = "" + if stderr is not None: + self.assertEqual(err.rstrip(), stderr) + try: + config = json.loads(out) + except json.JSONDecodeError: + self.fail(f"fail to decode stdout: {out!r}") + self.check_pre_config(config, expected_preconfig) self.check_core_config(config, expected_config) self.check_global_config(config) @@ -689,7 +694,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_config("init_read_set", core_config, preconfig, add_path="init_read_set_path") - def test_run_main_config(self): + def test_init_run_main(self): preconfig = {} code = ('import _testinternalcapi, json; ' 'print(json.dumps(_testinternalcapi.get_configs()))') @@ -699,7 +704,21 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'program_name': './python3', 'run_command': code + '\n', } - self.check_config("run_main_config", core_config, preconfig) + self.check_config("init_run_main", core_config, preconfig) + + def test_init_main(self): + preconfig = {} + code = ('import _testinternalcapi, json; ' + 'print(json.dumps(_testinternalcapi.get_configs()))') + core_config = { + 'argv': ['-c', 'arg2'], + 'program': 'python3', + 'program_name': './python3', + 'run_command': code + '\n', + '_init_main': 0, + } + self.check_config("init_main", core_config, preconfig, + stderr="Run Python code before _Py_InitializeMain") def test_init_dont_parse_argv(self): core_config = { diff --git a/Modules/getpath.c b/Modules/getpath.c index 3991ad719c1..34357e47bc2 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -493,7 +493,7 @@ calculate_prefix(const _PyCoreConfig *core_config, } if (!calculate->prefix_found) { - if (!core_config->_frozen) { + if (core_config->pathconfig_warnings) { fprintf(stderr, "Could not find platform independent libraries \n"); } @@ -681,7 +681,7 @@ calculate_exec_prefix(const _PyCoreConfig *core_config, } if (!calculate->exec_prefix_found) { - if (!core_config->_frozen) { + if (core_config->pathconfig_warnings) { fprintf(stderr, "Could not find platform dependent libraries \n"); } @@ -1206,7 +1206,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, } if ((!calculate->prefix_found || !calculate->exec_prefix_found) && - !core_config->_frozen) + core_config->pathconfig_warnings) { fprintf(stderr, "Consider setting $PYTHONHOME to [:]\n"); diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 4b2ed70de97..8cbbe17cfaf 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -83,7 +83,8 @@ main(int argc, char *argv[]) config.program_name = L"./_freeze_importlib"; /* Don't install importlib, since it could execute outdated bytecode. */ config._install_importlib = 0; - config._frozen = 1; + config.pathconfig_warnings = 0; + config._init_main = 0; _PyInitError err = _Py_InitializeFromConfig(&config); /* No need to call _PyCoreConfig_Clear() since we didn't allocate any diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 6eee2e8bd3e..4ee2cd1b407 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -757,17 +757,25 @@ fail: } -static int test_run_main(void) +wchar_t *init_main_argv[] = { + L"python3", L"-c", + (L"import _testinternalcapi, json; " + L"print(json.dumps(_testinternalcapi.get_configs()))"), + L"arg2"}; + + +static void configure_init_main(_PyCoreConfig *config) +{ + config->argv.length = Py_ARRAY_LENGTH(init_main_argv); + config->argv.items = init_main_argv; + config->program_name = L"./python3"; +} + + +static int test_init_run_main(void) { _PyCoreConfig config = _PyCoreConfig_INIT; - - wchar_t *argv[] = {L"python3", L"-c", - (L"import sys; " - L"print(f'_Py_RunMain(): sys.argv={sys.argv}')"), - L"arg2"}; - config.argv.length = Py_ARRAY_LENGTH(argv); - config.argv.items = argv; - config.program_name = L"./python3"; + configure_init_main(&config); _PyInitError err = _Py_InitializeFromConfig(&config); if (_Py_INIT_FAILED(err)) { @@ -778,13 +786,42 @@ static int test_run_main(void) } -static int test_run_main_config(void) +static int test_init_main(void) +{ + _PyCoreConfig config = _PyCoreConfig_INIT; + configure_init_main(&config); + config._init_main = 0; + + _PyInitError err = _Py_InitializeFromConfig(&config); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + /* sys.stdout don't exist yet: it is created by _Py_InitializeMain() */ + int res = PyRun_SimpleString( + "import sys; " + "print('Run Python code before _Py_InitializeMain', " + "file=sys.stderr)"); + if (res < 0) { + exit(1); + } + + err = _Py_InitializeMain(); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + return _Py_RunMain(); +} + + +static int test_run_main(void) { _PyCoreConfig config = _PyCoreConfig_INIT; wchar_t *argv[] = {L"python3", L"-c", - (L"import _testinternalcapi, json; " - L"print(json.dumps(_testinternalcapi.get_configs()))"), + (L"import sys; " + L"print(f'_Py_RunMain(): sys.argv={sys.argv}')"), L"arg2"}; config.argv.length = Py_ARRAY_LENGTH(argv); config.argv.items = argv; @@ -837,8 +874,9 @@ static struct TestCase TestCases[] = { { "preinit_isolated1", test_preinit_isolated1 }, { "preinit_isolated2", test_preinit_isolated2 }, { "init_read_set", test_init_read_set }, + { "init_run_main", test_init_run_main }, + { "init_main", test_init_main }, { "run_main", test_run_main }, - { "run_main_config", test_run_main_config }, { NULL, NULL } }; diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 2b13c5f0f9e..8a5e5d509cb 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -667,7 +667,8 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_WSTR_ATTR(run_module); COPY_WSTR_ATTR(run_filename); COPY_WSTR_ATTR(check_hash_pycs_mode); - COPY_ATTR(_frozen); + COPY_ATTR(pathconfig_warnings); + COPY_ATTR(_init_main); #undef COPY_ATTR #undef COPY_WSTR_ATTR @@ -766,7 +767,8 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_WSTR(run_filename); SET_ITEM_INT(_install_importlib); SET_ITEM_WSTR(check_hash_pycs_mode); - SET_ITEM_INT(_frozen); + SET_ITEM_INT(pathconfig_warnings); + SET_ITEM_INT(_init_main); return dict; @@ -855,7 +857,7 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) #ifdef MS_WINDOWS COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); #endif - COPY_FLAG(_frozen, Py_FrozenFlag); + COPY_NOT_FLAG(pathconfig_warnings, Py_FrozenFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); COPY_NOT_FLAG(site_import, Py_NoSiteFlag); @@ -892,7 +894,7 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) #ifdef MS_WINDOWS COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); #endif - COPY_FLAG(_frozen, Py_FrozenFlag); + COPY_NOT_FLAG(pathconfig_warnings, Py_FrozenFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); COPY_NOT_FLAG(site_import, Py_NoSiteFlag); @@ -2253,7 +2255,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) assert(!(config->run_command != NULL && config->run_module != NULL)); assert(config->check_hash_pycs_mode != NULL); assert(config->_install_importlib >= 0); - assert(config->_frozen >= 0); + assert(config->pathconfig_warnings >= 0); err = _Py_INIT_OK(); diff --git a/Python/frozenmain.c b/Python/frozenmain.c index a777576ad78..f2499ef84cd 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -40,7 +40,7 @@ Py_FrozenMain(int argc, char **argv) } _PyCoreConfig config = _PyCoreConfig_INIT; - config._frozen = 1; /* Suppress errors from getpath.c */ + config.pathconfig_warnings = 0; /* Suppress errors from getpath.c */ if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') inspect = 1; diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index a173eb380a5..e89152637fe 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -970,6 +970,21 @@ _Py_InitializeMainInterpreter(_PyRuntimeState *runtime, return _Py_INIT_OK(); } + +_PyInitError +_Py_InitializeMain(void) +{ + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + _PyRuntimeState *runtime = &_PyRuntime; + PyInterpreterState *interp = _PyRuntimeState_GetThreadState(runtime)->interp; + + return _Py_InitializeMainInterpreter(runtime, interp); +} + + #undef _INIT_DEBUG_PRINT static _PyInitError @@ -990,7 +1005,7 @@ init_python(const _PyCoreConfig *config, const _PyArgv *args) } config = &interp->core_config; - if (!config->_frozen) { + if (config->_init_main) { err = _Py_InitializeMainInterpreter(runtime, interp); if (_Py_INIT_FAILED(err)) { return err; diff --git a/Python/pythonrun.c b/Python/pythonrun.c index 3d83044af9a..bc131fd7e5e 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -1046,6 +1046,15 @@ run_eval_code_obj(PyCodeObject *co, PyObject *globals, PyObject *locals) * Py_Main() based one. */ _Py_UnhandledKeyboardInterrupt = 0; + + /* Set globals['__builtins__'] if it doesn't exist */ + if (globals != NULL && PyDict_GetItemString(globals, "__builtins__") == NULL) { + PyInterpreterState *interp = _PyInterpreterState_Get(); + if (PyDict_SetItemString(globals, "__builtins__", interp->builtins) < 0) { + return NULL; + } + } + v = PyEval_EvalCode((PyObject*)co, globals, locals); if (!v && PyErr_Occurred() == PyExc_KeyboardInterrupt) { _Py_UnhandledKeyboardInterrupt = 1;