From 6d1c46746e17367caf8a24623cb5c9a9c4e3e036 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 May 2019 11:02:00 +0200 Subject: [PATCH] bpo-36763: Fix Python preinitialization (GH-13432) * Add _PyPreConfig.parse_argv * Add _PyCoreConfig._config_init field and _PyCoreConfigInitEnum enum type * Initialization functions: reject preconfig=NULL and config=NULL * Add config parameter to _PyCoreConfig_DecodeLocaleErr(): pass config->argv to _Py_PreInitializeFromPyArgv(), to parse config command line arguments in preinitialization. * Add config parameter to _PyCoreConfig_SetString(). It now preinitializes Python. * _PyCoreConfig_SetPyArgv() now also preinitializes Python for wide argv * Fix _Py_PreInitializeFromCoreConfig(): don't pass args to _Py_PreInitializeFromPyArgv() if config.parse_argv=0. * Use "char * const *" and "wchar_t * const *" types for 'argv' parameters and _PyArgv.argv. * Add unit test on preinitialization from argv. * _PyPreConfig.allocator type becomes int * Add _PyPreConfig_InitFromPreConfig() and _PyPreConfig_InitFromCoreConfig() helper functions --- Include/cpython/coreconfig.h | 22 ++- Include/cpython/pylifecycle.h | 4 +- Include/internal/pycore_coreconfig.h | 10 +- Lib/test/test_embed.py | 96 +++++++--- Programs/_testembed.c | 264 ++++++++++++++++++++++----- Python/coreconfig.c | 123 +++++++------ Python/preconfig.c | 80 +++++--- Python/pylifecycle.c | 57 ++++-- 8 files changed, 477 insertions(+), 179 deletions(-) diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index a71f16171b7..decfb70e734 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -44,6 +44,10 @@ typedef struct { int _config_version; /* Internal configuration version, used for ABI compatibility */ + /* Parse _Py_PreInitializeFromArgs() arguments? + See _PyCoreConfig.parse_argv */ + int parse_argv; + /* If greater than 0, enable isolated mode: sys.path contains neither the script's directory nor the user's site-packages directory. @@ -111,8 +115,9 @@ typedef struct { int dev_mode; /* Development mode. PYTHONDEVMODE, -X dev */ - /* Memory allocator: PYTHONMALLOC env var */ - PyMemAllocatorName allocator; + /* Memory allocator: PYTHONMALLOC env var. + See PyMemAllocatorName for valid values. */ + int allocator; } _PyPreConfig; PyAPI_FUNC(void) _PyPreConfig_InitPythonConfig(_PyPreConfig *config); @@ -121,9 +126,16 @@ PyAPI_FUNC(void) _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config); /* --- _PyCoreConfig ---------------------------------------------- */ +typedef enum { + _PyCoreConfig_INIT = 0, + _PyCoreConfig_INIT_PYTHON = 1, + _PyCoreConfig_INIT_ISOLATED = 2 +} _PyCoreConfigInitEnum; + typedef struct { int _config_version; /* Internal configuration version, used for ABI compatibility */ + int _config_init; /* _PyCoreConfigInitEnum value */ int isolated; /* Isolated mode? see _PyPreConfig.isolated */ int use_environment; /* Use environment variables? see _PyPreConfig.use_environment */ @@ -401,19 +413,21 @@ PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitPythonConfig(_PyCoreConfig *config); PyAPI_FUNC(_PyInitError) _PyCoreConfig_InitIsolatedConfig(_PyCoreConfig *config); PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetString( + _PyCoreConfig *config, wchar_t **config_str, const wchar_t *str); PyAPI_FUNC(_PyInitError) _PyCoreConfig_DecodeLocale( + _PyCoreConfig *config, wchar_t **config_str, const char *str); PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetArgv( _PyCoreConfig *config, Py_ssize_t argc, - char **argv); + char * const *argv); PyAPI_FUNC(_PyInitError) _PyCoreConfig_SetWideArgv(_PyCoreConfig *config, Py_ssize_t argc, - wchar_t **argv); + wchar_t * const *argv); #ifdef __cplusplus } diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index 8fc809d30d8..1e1dabe59ff 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -35,11 +35,11 @@ PyAPI_FUNC(_PyInitError) _Py_InitializeFromConfig( PyAPI_FUNC(_PyInitError) _Py_InitializeFromArgs( const _PyCoreConfig *config, Py_ssize_t argc, - char **argv); + char * const *argv); PyAPI_FUNC(_PyInitError) _Py_InitializeFromWideArgs( const _PyCoreConfig *config, Py_ssize_t argc, - wchar_t **argv); + wchar_t * const *argv); PyAPI_FUNC(_PyInitError) _Py_InitializeMain(void); PyAPI_FUNC(int) _Py_RunMain(void); diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h index edde7b1d8d8..324e0b82b90 100644 --- a/Include/internal/pycore_coreconfig.h +++ b/Include/internal/pycore_coreconfig.h @@ -63,8 +63,8 @@ PyAPI_FUNC(int) _PyWstrList_Extend(_PyWstrList *list, typedef struct { Py_ssize_t argc; int use_bytes_argv; - char **bytes_argv; - wchar_t **wchar_argv; + char * const *bytes_argv; + wchar_t * const *wchar_argv; } _PyArgv; PyAPI_FUNC(_PyInitError) _PyArgv_AsWstrList(const _PyArgv *args, @@ -121,6 +121,12 @@ PyAPI_FUNC(_PyInitError) _PyPreCmdline_Read(_PyPreCmdline *cmdline, /* --- _PyPreConfig ----------------------------------------------- */ PyAPI_FUNC(void) _PyPreConfig_Init(_PyPreConfig *config); +PyAPI_FUNC(void) _PyPreConfig_InitFromCoreConfig( + _PyPreConfig *config, + const _PyCoreConfig *coreconfig); +PyAPI_FUNC(void) _PyPreConfig_InitFromPreConfig( + _PyPreConfig *config, + const _PyPreConfig *config2); PyAPI_FUNC(void) _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2); PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 5a5419dcfcf..5be179a266b 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -17,6 +17,10 @@ PYMEM_ALLOCATOR_NOT_SET = 0 PYMEM_ALLOCATOR_DEBUG = 2 PYMEM_ALLOCATOR_MALLOC = 3 +CONFIG_INIT = 0 +CONFIG_INIT_PYTHON = 1 +CONFIG_INIT_ISOLATED = 2 + class EmbeddingTestsMixin: def setUp(self): @@ -280,11 +284,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): DEFAULT_PRE_CONFIG = { 'allocator': PYMEM_ALLOCATOR_NOT_SET, + 'parse_argv': 0, 'configure_locale': 1, 'coerce_c_locale': 0, 'coerce_c_locale_warn': 0, 'utf8_mode': 0, } + if MS_WINDOWS: + DEFAULT_PRE_CONFIG.update({ + 'legacy_windows_fs_encoding': 0, + }) + PYTHON_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG, + parse_argv=1, + ) ISOLATED_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG, configure_locale=0, isolated=1, @@ -292,8 +304,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): utf8_mode=0, dev_mode=0, ) - if MS_WINDOWS: - ISOLATED_PRE_CONFIG['legacy_windows_fs_encoding'] = 0 COPY_PRE_CONFIG = [ 'dev_mode', @@ -302,6 +312,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ] DEFAULT_CORE_CONFIG = { + '_config_init': CONFIG_INIT, 'isolated': 0, 'use_environment': 1, 'dev_mode': 0, @@ -365,9 +376,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_init_main': 1, } if MS_WINDOWS: - DEFAULT_PRE_CONFIG.update({ - 'legacy_windows_fs_encoding': 0, - }) DEFAULT_CORE_CONFIG.update({ 'legacy_windows_stdio': 0, }) @@ -439,13 +447,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): def get_expected_config(self, expected_preconfig, expected, env, api, add_path=None): - if api == "python": + if api == CONFIG_INIT_PYTHON: default_config = self.PYTHON_CORE_CONFIG - elif api == "isolated": + elif api == CONFIG_INIT_ISOLATED: default_config = self.ISOLATED_CORE_CONFIG else: default_config = self.DEFAULT_CORE_CONFIG expected = dict(default_config, **expected) + expected['_config_init'] = api code = textwrap.dedent(''' import json @@ -519,9 +528,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): return expected def check_pre_config(self, config, expected): - pre_config = dict(config['pre_config']) - core_config = dict(config['core_config']) - self.assertEqual(pre_config, expected) + self.assertEqual(config['pre_config'], expected) def check_core_config(self, config, expected): core_config = dict(config['core_config']) @@ -554,7 +561,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(config['global_config'], expected) def check_config(self, testname, expected_config=None, expected_preconfig=None, - add_path=None, stderr=None, api="default"): + add_path=None, stderr=None, api=CONFIG_INIT): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment for key in list(env): @@ -565,8 +572,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): env['PYTHONCOERCECLOCALE'] = '0' env['PYTHONUTF8'] = '0' - if api == "isolated": + if api == CONFIG_INIT_ISOLATED: default_preconfig = self.ISOLATED_PRE_CONFIG + elif api == CONFIG_INIT_PYTHON: + default_preconfig = self.PYTHON_PRE_CONFIG else: default_preconfig = self.DEFAULT_PRE_CONFIG if expected_preconfig is None: @@ -719,7 +728,38 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'dev_mode': 1, 'warnoptions': ['default'], } - self.check_config("init_dev_mode", config, preconfig, api="python") + self.check_config("init_dev_mode", config, preconfig, + api=CONFIG_INIT_PYTHON) + + def test_preinit_parse_argv(self): + # Pre-initialize implicitly using argv: make sure that -X dev + # is used to configure the allocation in preinitialization + preconfig = { + 'allocator': PYMEM_ALLOCATOR_DEBUG, + } + config = { + 'argv': ['script.py'], + 'run_filename': 'script.py', + 'dev_mode': 1, + 'faulthandler': 1, + 'warnoptions': ['default'], + 'xoptions': ['dev'], + } + self.check_config("preinit_parse_argv", config, preconfig, + api=CONFIG_INIT_PYTHON) + + def test_preinit_dont_parse_argv(self): + # -X dev must be ignored by isolated preconfiguration + preconfig = { + 'isolated': 0, + } + config = { + 'argv': ["python3", "-E", "-I", + "-X", "dev", "-X", "utf8", "script.py"], + 'isolated': 0, + } + self.check_config("preinit_dont_parse_argv", config, preconfig, + api=CONFIG_INIT_ISOLATED) def test_init_isolated_flag(self): config = { @@ -727,7 +767,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'use_environment': 0, 'user_site_directory': 0, } - self.check_config("init_isolated_flag", config, api="python") + self.check_config("init_isolated_flag", config, api=CONFIG_INIT_PYTHON) def test_preinit_isolated1(self): # _PyPreConfig.isolated=1, _PyCoreConfig.isolated not set @@ -747,25 +787,30 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): } self.check_config("preinit_isolated2", config) + def test_preinit_isolated_config(self): + self.check_config("preinit_isolated_config", api=CONFIG_INIT_ISOLATED) + def test_init_isolated_config(self): - self.check_config("init_isolated_config", api="isolated") + self.check_config("init_isolated_config", api=CONFIG_INIT_ISOLATED) def test_init_python_config(self): - self.check_config("init_python_config", api="python") + self.check_config("init_python_config", api=CONFIG_INIT_PYTHON) def test_init_dont_configure_locale(self): # _PyPreConfig.configure_locale=0 preconfig = { 'configure_locale': 0, } - self.check_config("init_dont_configure_locale", {}, preconfig, api="python") + self.check_config("init_dont_configure_locale", {}, preconfig, + api=CONFIG_INIT_PYTHON) def test_init_read_set(self): core_config = { 'program_name': './init_read_set', 'executable': 'my_executable', } - self.check_config("init_read_set", core_config, api="python", + self.check_config("init_read_set", core_config, + api=CONFIG_INIT_PYTHON, add_path="init_read_set_path") def test_init_run_main(self): @@ -777,7 +822,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'run_command': code + '\n', 'parse_argv': 1, } - self.check_config("init_run_main", core_config, api="python") + self.check_config("init_run_main", core_config, + api=CONFIG_INIT_PYTHON) def test_init_main(self): code = ('import _testinternalcapi, json; ' @@ -789,7 +835,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'parse_argv': 1, '_init_main': 0, } - self.check_config("init_main", core_config, api="python", + self.check_config("init_main", core_config, + api=CONFIG_INIT_PYTHON, stderr="Run Python code before _Py_InitializeMain") def test_init_parse_argv(self): @@ -800,15 +847,20 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'run_command': 'pass\n', 'use_environment': 0, } - self.check_config("init_parse_argv", core_config, api="python") + self.check_config("init_parse_argv", core_config, + api=CONFIG_INIT_PYTHON) def test_init_dont_parse_argv(self): + pre_config = { + 'parse_argv': 0, + } core_config = { 'parse_argv': 0, 'argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', } - self.check_config("init_dont_parse_argv", core_config, api="python") + self.check_config("init_dont_parse_argv", core_config, pre_config, + api=CONFIG_INIT_PYTHON) if __name__ == "__main__": diff --git a/Programs/_testembed.c b/Programs/_testembed.c index f1bb731dcba..3dabf66de15 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -527,7 +527,10 @@ static int check_init_parse_argv(int parse_argv) _PyInitError err; _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } static wchar_t* argv[] = { L"./argv0", @@ -565,7 +568,7 @@ static int test_init_dont_parse_argv(void) } -static void test_init_env_putenvs(void) +static void set_all_env_vars(void) { putenv("PYTHONHASHSEED=42"); putenv("PYTHONMALLOC=malloc"); @@ -596,7 +599,7 @@ static int test_init_env(void) { /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; - test_init_env_putenvs(); + set_all_env_vars(); _testembed_Py_Initialize(); dump_config(); Py_Finalize(); @@ -604,9 +607,9 @@ static int test_init_env(void) } -static void set_all_env_vars(void) +static void set_all_env_vars_dev_mode(void) { - test_init_env_putenvs(); + set_all_env_vars(); putenv("PYTHONMALLOC="); putenv("PYTHONFAULTHANDLER="); putenv("PYTHONDEVMODE=1"); @@ -617,7 +620,7 @@ static int test_init_env_dev_mode(void) { /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; - set_all_env_vars(); + set_all_env_vars_dev_mode(); _testembed_Py_Initialize(); dump_config(); Py_Finalize(); @@ -629,7 +632,7 @@ static int test_init_env_dev_mode_alloc(void) { /* Test initialization from environment variables */ Py_IgnoreEnvironmentFlag = 0; - set_all_env_vars(); + set_all_env_vars_dev_mode(); putenv("PYTHONMALLOC=malloc"); _testembed_Py_Initialize(); dump_config(); @@ -644,7 +647,10 @@ static int test_init_isolated_flag(void) /* Test _PyCoreConfig.isolated=1 */ _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } Py_IsolatedFlag = 0; config.isolated = 1; @@ -727,6 +733,107 @@ static int test_preinit_isolated2(void) } +static int test_preinit_dont_parse_argv(void) +{ + _PyInitError err; + + _PyPreConfig preconfig; + _PyPreConfig_InitIsolatedConfig(&preconfig); + + preconfig.isolated = 0; + + /* -X dev must be ignored by isolated preconfiguration */ + wchar_t *argv[] = {L"python3", + L"-E", + L"-I", + L"-X", L"dev", + L"-X", L"utf8", + L"script.py"}; + err = _Py_PreInitializeFromWideArgs(&preconfig, Py_ARRAY_LENGTH(argv), argv); + if (_PyInitError_Failed(err)) { + goto failed; + } + + _PyCoreConfig config; + + err = _PyCoreConfig_InitIsolatedConfig(&config); + if (_PyInitError_Failed(err)) { + goto failed; + } + + config.isolated = 0; + + /* Pre-initialize implicitly using argv: make sure that -X dev + is used to configure the allocation in preinitialization */ + err = _PyCoreConfig_SetWideArgv(&config, Py_ARRAY_LENGTH(argv), argv); + if (_PyInitError_Failed(err)) { + goto failed; + } + + err = _PyCoreConfig_SetString(&config, &config.program_name, + L"./_testembed"); + if (_PyInitError_Failed(err)) { + goto failed; + } + + err = _Py_InitializeFromConfig(&config); + if (_PyInitError_Failed(err)) { + goto failed; + } + _PyCoreConfig_Clear(&config); + + dump_config(); + Py_Finalize(); + return 0; + +failed: + _PyCoreConfig_Clear(&config); + _Py_ExitInitError(err); +} + + +static int test_preinit_parse_argv(void) +{ + _PyInitError err; + _PyCoreConfig config; + + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + goto failed; + } + + /* Pre-initialize implicitly using argv: make sure that -X dev + is used to configure the allocation in preinitialization */ + wchar_t *argv[] = {L"python3", L"-X", L"dev", L"script.py"}; + err = _PyCoreConfig_SetWideArgv(&config, Py_ARRAY_LENGTH(argv), argv); + if (_PyInitError_Failed(err)) { + goto failed; + } + + err = _PyCoreConfig_SetString(&config, &config.program_name, + L"./_testembed"); + if (_PyInitError_Failed(err)) { + goto failed; + } + + err = _Py_InitializeFromConfig(&config); + if (_PyInitError_Failed(err)) { + goto failed; + } + _PyCoreConfig_Clear(&config); + + dump_config(); + Py_Finalize(); + return 0; + +failed: + _PyCoreConfig_Clear(&config); + _Py_ExitInitError(err); +} + + + + static void set_all_global_config_variables(void) { Py_IsolatedFlag = 0; @@ -749,9 +856,10 @@ static void set_all_global_config_variables(void) } -static int test_init_isolated_config(void) +static int check_preinit_isolated_config(int preinit) { _PyInitError err; + _PyPreConfig *rt_preconfig; /* environment variables must be ignored */ set_all_env_vars(); @@ -759,18 +867,20 @@ static int test_init_isolated_config(void) /* global configuration variables must be ignored */ set_all_global_config_variables(); - _PyPreConfig preconfig; - _PyPreConfig_InitIsolatedConfig(&preconfig); + if (preinit) { + _PyPreConfig preconfig; + _PyPreConfig_InitIsolatedConfig(&preconfig); - err = _Py_PreInitialize(&preconfig); - if (_PyInitError_Failed(err)) { - _Py_ExitInitError(err); + err = _Py_PreInitialize(&preconfig); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } + + rt_preconfig = &_PyRuntime.preconfig; + assert(rt_preconfig->isolated == 1); + assert(rt_preconfig->use_environment == 0); } - _PyPreConfig *rt_preconfig = &_PyRuntime.preconfig; - assert(rt_preconfig->isolated == 1); - assert(rt_preconfig->use_environment == 0); - _PyCoreConfig config; err = _PyCoreConfig_InitIsolatedConfig(&config); if (_PyInitError_Failed(err)) { @@ -782,13 +892,30 @@ static int test_init_isolated_config(void) if (_PyInitError_Failed(err)) { _Py_ExitInitError(err); } + + rt_preconfig = &_PyRuntime.preconfig; + assert(rt_preconfig->isolated == 1); + assert(rt_preconfig->use_environment == 0); + dump_config(); Py_Finalize(); return 0; } -static int test_init_python_config(void) +static int test_preinit_isolated_config(void) +{ + return check_preinit_isolated_config(1); +} + + +static int test_init_isolated_config(void) +{ + return check_preinit_isolated_config(0); +} + + +static int check_init_python_config(int preinit) { _PyInitError err; @@ -805,12 +932,14 @@ static int test_init_python_config(void) Py_LegacyWindowsStdioFlag = 1; #endif - _PyPreConfig preconfig; - _PyPreConfig_InitPythonConfig(&preconfig); + if (preinit) { + _PyPreConfig preconfig; + _PyPreConfig_InitPythonConfig(&preconfig); - err = _Py_PreInitialize(&preconfig); - if (_PyInitError_Failed(err)) { - _Py_ExitInitError(err); + err = _Py_PreInitialize(&preconfig); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } } _PyCoreConfig config; @@ -830,6 +959,18 @@ static int test_init_python_config(void) } +static int test_preinit_python_config(void) +{ + return check_init_python_config(1); +} + + +static int test_init_python_config(void) +{ + return check_init_python_config(0); +} + + static int test_init_dont_configure_locale(void) { _PyInitError err; @@ -846,7 +987,10 @@ static int test_init_dont_configure_locale(void) } _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } config.program_name = L"./_testembed"; err = _Py_InitializeFromConfig(&config); if (_PyInitError_Failed(err)) { @@ -861,13 +1005,17 @@ static int test_init_dont_configure_locale(void) static int test_init_dev_mode(void) { + _PyInitError err; _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); config.dev_mode = 1; config.program_name = L"./_testembed"; - _PyInitError err = _Py_InitializeFromConfig(&config); + err = _Py_InitializeFromConfig(&config); if (_PyInitError_Failed(err)) { _Py_ExitInitError(err); } @@ -881,9 +1029,13 @@ static int test_init_read_set(void) { _PyInitError err; _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } - err = _PyCoreConfig_DecodeLocale(&config.program_name, "./init_read_set"); + err = _PyCoreConfig_DecodeLocale(&config, &config.program_name, + "./init_read_set"); if (_PyInitError_Failed(err)) { goto fail; } @@ -900,7 +1052,7 @@ static int test_init_read_set(void) } /* override executable computed by _PyCoreConfig_Read() */ - err = _PyCoreConfig_SetString(&config.executable, L"my_executable"); + err = _PyCoreConfig_SetString(&config, &config.executable, L"my_executable"); if (_PyInitError_Failed(err)) { goto fail; } @@ -937,11 +1089,15 @@ static void configure_init_main(_PyCoreConfig *config) static int test_init_run_main(void) { + _PyInitError err; _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } configure_init_main(&config); - _PyInitError err = _Py_InitializeFromConfig(&config); + err = _Py_InitializeFromConfig(&config); if (_PyInitError_Failed(err)) { _Py_ExitInitError(err); } @@ -952,12 +1108,17 @@ static int test_init_run_main(void) static int test_init_main(void) { + _PyInitError err; _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + _Py_ExitInitError(err); + } configure_init_main(&config); config._init_main = 0; - _PyInitError err = _Py_InitializeFromConfig(&config); + err = _Py_InitializeFromConfig(&config); if (_PyInitError_Failed(err)) { _Py_ExitInitError(err); } @@ -982,23 +1143,40 @@ static int test_init_main(void) static int test_run_main(void) { + _PyInitError err; _PyCoreConfig config; - _PyCoreConfig_InitPythonConfig(&config); + + err = _PyCoreConfig_InitPythonConfig(&config); + if (_PyInitError_Failed(err)) { + goto failed; + } wchar_t *argv[] = {L"python3", L"-c", (L"import sys; " L"print(f'_Py_RunMain(): sys.argv={sys.argv}')"), L"arg2"}; - config.argv.length = Py_ARRAY_LENGTH(argv); - config.argv.items = argv; - config.program_name = L"./python3"; - - _PyInitError err = _Py_InitializeFromConfig(&config); + err = _PyCoreConfig_SetWideArgv(&config, Py_ARRAY_LENGTH(argv), argv); if (_PyInitError_Failed(err)) { - _Py_ExitInitError(err); + goto failed; } + err = _PyCoreConfig_SetString(&config, &config.program_name, + L"./python3"); + if (_PyInitError_Failed(err)) { + goto failed; + } + + err = _Py_InitializeFromConfig(&config); + if (_PyInitError_Failed(err)) { + goto failed; + } + _PyCoreConfig_Clear(&config); + return _Py_RunMain(); + +failed: + _PyCoreConfig_Clear(&config); + _Py_ExitInitError(err); } @@ -1039,10 +1217,14 @@ static struct TestCase TestCases[] = { { "init_dont_configure_locale", test_init_dont_configure_locale }, { "init_dev_mode", test_init_dev_mode }, { "init_isolated_flag", test_init_isolated_flag }, + { "preinit_isolated_config", test_preinit_isolated_config }, { "init_isolated_config", test_init_isolated_config }, + { "preinit_python_config", test_preinit_python_config }, { "init_python_config", test_init_python_config }, { "preinit_isolated1", test_preinit_isolated1 }, { "preinit_isolated2", test_preinit_isolated2 }, + { "preinit_parse_argv", test_preinit_parse_argv }, + { "preinit_dont_parse_argv", test_preinit_dont_parse_argv }, { "init_read_set", test_init_read_set }, { "init_run_main", test_init_run_main }, { "init_main", test_init_main }, diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 470bda87028..958845e488d 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -551,6 +551,7 @@ _PyCoreConfig_Init(_PyCoreConfig *config) memset(config, 0, sizeof(*config)); config->_config_version = _Py_CONFIG_VERSION; + config->_config_init = (int)_PyCoreConfig_INIT; config->isolated = -1; config->use_environment = -1; config->dev_mode = -1; @@ -612,6 +613,7 @@ _PyCoreConfig_InitPythonConfig(_PyCoreConfig *config) { _PyCoreConfig_InitDefaults(config); + config->_config_init = (int)_PyCoreConfig_INIT_PYTHON; config->configure_c_stdio = 1; config->parse_argv = 1; @@ -624,6 +626,7 @@ _PyCoreConfig_InitIsolatedConfig(_PyCoreConfig *config) { _PyCoreConfig_InitDefaults(config); + config->_config_init = (int)_PyCoreConfig_INIT_ISOLATED; config->isolated = 1; config->use_environment = 0; config->user_site_directory = 0; @@ -643,8 +646,14 @@ _PyCoreConfig_InitIsolatedConfig(_PyCoreConfig *config) /* Copy str into *config_str (duplicate the string) */ _PyInitError -_PyCoreConfig_SetString(wchar_t **config_str, const wchar_t *str) +_PyCoreConfig_SetString(_PyCoreConfig *config, wchar_t **config_str, + const wchar_t *str) { + _PyInitError err = _Py_PreInitializeFromCoreConfig(config, NULL); + if (_Py_INIT_FAILED(err)) { + return err; + } + wchar_t *str2; if (str != NULL) { str2 = _PyMem_RawWcsdup(str); @@ -662,10 +671,10 @@ _PyCoreConfig_SetString(wchar_t **config_str, const wchar_t *str) static _PyInitError -_PyCoreConfig_DecodeLocaleErr(wchar_t **config_str, const char *str, - const char *decode_err_msg) +_PyCoreConfig_DecodeLocaleErr(_PyCoreConfig *config, wchar_t **config_str, + const char *str, const char *decode_err_msg) { - _PyInitError err = _Py_PreInitialize(NULL); + _PyInitError err = _Py_PreInitializeFromCoreConfig(config, NULL); if (_Py_INIT_FAILED(err)) { return err; } @@ -692,17 +701,18 @@ _PyCoreConfig_DecodeLocaleErr(wchar_t **config_str, const char *str, } -#define CONFIG_DECODE_LOCALE(config_str, str, NAME) \ - _PyCoreConfig_DecodeLocaleErr(config_str, str, "cannot decode " NAME) +#define CONFIG_DECODE_LOCALE(config, config_str, str, NAME) \ + _PyCoreConfig_DecodeLocaleErr(config, config_str, str, "cannot decode " NAME) /* Decode str using Py_DecodeLocale() and set the result into *config_str. Pre-initialize Python if needed to ensure that encodings are properly configured. */ _PyInitError -_PyCoreConfig_DecodeLocale(wchar_t **config_str, const char *str) +_PyCoreConfig_DecodeLocale(_PyCoreConfig *config, wchar_t **config_str, + const char *str) { - return CONFIG_DECODE_LOCALE(config_str, str, "string"); + return CONFIG_DECODE_LOCALE(config, config_str, str, "string"); } @@ -715,7 +725,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_WSTR_ATTR(ATTR) \ do { \ - err = _PyCoreConfig_SetString(&config->ATTR, config2->ATTR); \ + err = _PyCoreConfig_SetString(config, &config->ATTR, config2->ATTR); \ if (_Py_INIT_FAILED(err)) { \ return err; \ } \ @@ -727,6 +737,7 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) } \ } while (0) + COPY_ATTR(_config_init); COPY_ATTR(isolated); COPY_ATTR(use_environment); COPY_ATTR(dev_mode); @@ -829,6 +840,7 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) #define SET_ITEM_WSTRLIST(LIST) \ SET_ITEM(#LIST, _PyWstrList_AsList(&config->LIST)) + SET_ITEM_INT(_config_init); SET_ITEM_INT(isolated); SET_ITEM_INT(use_environment); SET_ITEM_INT(dev_mode); @@ -910,7 +922,7 @@ _PyCoreConfig_GetEnv(const _PyCoreConfig *config, const char *name) Return 0 on success, but *dest can be NULL. Return -1 on memory allocation failure. Return -2 on decoding error. */ static _PyInitError -_PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, +_PyCoreConfig_GetEnvDup(_PyCoreConfig *config, wchar_t **dest, wchar_t *wname, char *name, const char *decode_err_msg) @@ -930,7 +942,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, return _Py_INIT_OK(); } - return _PyCoreConfig_SetString(dest, var); + return _PyCoreConfig_SetString(config, dest, var); #else const char *var = getenv(name); if (!var || var[0] == '\0') { @@ -938,7 +950,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, return _Py_INIT_OK(); } - return _PyCoreConfig_DecodeLocaleErr(dest, var, decode_err_msg); + return _PyCoreConfig_DecodeLocaleErr(config, dest, var, decode_err_msg); #endif } @@ -1053,7 +1065,7 @@ config_init_program_name(_PyCoreConfig *config) script. */ const char *p = _PyCoreConfig_GetEnv(config, "PYTHONEXECUTABLE"); if (p != NULL) { - err = CONFIG_DECODE_LOCALE(&config->program_name, p, + err = CONFIG_DECODE_LOCALE(config, &config->program_name, p, "PYTHONEXECUTABLE environment variable"); if (_Py_INIT_FAILED(err)) { return err; @@ -1067,7 +1079,8 @@ config_init_program_name(_PyCoreConfig *config) /* Used by Mac/Tools/pythonw.c to forward * the argv0 of the stub executable */ - err = CONFIG_DECODE_LOCALE(&config->program_name, pyvenv_launcher, + err = CONFIG_DECODE_LOCALE(config, + &config->program_name, pyvenv_launcher, "__PYVENV_LAUNCHER__ environment variable"); if (_Py_INIT_FAILED(err)) { return err; @@ -1094,7 +1107,8 @@ config_init_program_name(_PyCoreConfig *config) #else const wchar_t *default_program_name = L"python3"; #endif - err = _PyCoreConfig_SetString(&config->program_name, default_program_name); + err = _PyCoreConfig_SetString(config, &config->program_name, + default_program_name); if (_Py_INIT_FAILED(err)) { return err; } @@ -1109,7 +1123,8 @@ config_init_executable(_PyCoreConfig *config) /* If Py_SetProgramFullPath() was called, use its value */ const wchar_t *program_full_path = _Py_path_config.program_full_path; if (program_full_path != NULL) { - _PyInitError err = _PyCoreConfig_SetString(&config->executable, + _PyInitError err = _PyCoreConfig_SetString(config, + &config->executable, program_full_path); if (_Py_INIT_FAILED(err)) { return err; @@ -1135,7 +1150,7 @@ config_init_home(_PyCoreConfig *config) /* If Py_SetPythonHome() was called, use its value */ wchar_t *home = _Py_path_config.home; if (home) { - _PyInitError err = _PyCoreConfig_SetString(&config->home, home); + _PyInitError err = _PyCoreConfig_SetString(config, &config->home, home); if (_Py_INIT_FAILED(err)) { return err; } @@ -1392,14 +1407,14 @@ config_get_stdio_errors(const _PyCoreConfig *config) static _PyInitError -config_get_locale_encoding(wchar_t **locale_encoding) +config_get_locale_encoding(_PyCoreConfig *config, wchar_t **locale_encoding) { #ifdef MS_WINDOWS char encoding[20]; PyOS_snprintf(encoding, sizeof(encoding), "cp%u", GetACP()); - return _PyCoreConfig_DecodeLocale(locale_encoding, encoding); + return _PyCoreConfig_DecodeLocale(config, locale_encoding, encoding); #elif defined(_Py_FORCE_UTF8_LOCALE) - return _PyCoreConfig_SetString(locale_encoding, L"utf-8"); + return _PyCoreConfig_SetString(config, locale_encoding, L"utf-8"); #else const char *encoding = nl_langinfo(CODESET); if (!encoding || encoding[0] == '\0') { @@ -1407,7 +1422,8 @@ config_get_locale_encoding(wchar_t **locale_encoding) "nl_langinfo(CODESET) failed"); } /* nl_langinfo(CODESET) is decoded by Py_DecodeLocale() */ - return CONFIG_DECODE_LOCALE(locale_encoding, encoding, + return CONFIG_DECODE_LOCALE(config, + locale_encoding, encoding, "nl_langinfo(CODESET)"); #endif } @@ -1422,7 +1438,7 @@ config_init_stdio_encoding(_PyCoreConfig *config, /* If Py_SetStandardStreamEncoding() have been called, use these parameters. */ if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { - err = CONFIG_DECODE_LOCALE(&config->stdio_encoding, + err = CONFIG_DECODE_LOCALE(config, &config->stdio_encoding, _Py_StandardStreamEncoding, "_Py_StandardStreamEncoding"); if (_Py_INIT_FAILED(err)) { @@ -1431,7 +1447,7 @@ config_init_stdio_encoding(_PyCoreConfig *config, } if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { - err = CONFIG_DECODE_LOCALE(&config->stdio_errors, + err = CONFIG_DECODE_LOCALE(config, &config->stdio_errors, _Py_StandardStreamErrors, "_Py_StandardStreamErrors"); if (_Py_INIT_FAILED(err)) { @@ -1463,7 +1479,7 @@ config_init_stdio_encoding(_PyCoreConfig *config, /* Does PYTHONIOENCODING contain an encoding? */ if (pythonioencoding[0]) { if (config->stdio_encoding == NULL) { - err = CONFIG_DECODE_LOCALE(&config->stdio_encoding, + err = CONFIG_DECODE_LOCALE(config, &config->stdio_encoding, pythonioencoding, "PYTHONIOENCODING environment variable"); if (_Py_INIT_FAILED(err)) { @@ -1482,7 +1498,7 @@ config_init_stdio_encoding(_PyCoreConfig *config, } if (config->stdio_errors == NULL && errors != NULL) { - err = CONFIG_DECODE_LOCALE(&config->stdio_errors, + err = CONFIG_DECODE_LOCALE(config, &config->stdio_errors, errors, "PYTHONIOENCODING environment variable"); if (_Py_INIT_FAILED(err)) { @@ -1497,13 +1513,13 @@ config_init_stdio_encoding(_PyCoreConfig *config, /* UTF-8 Mode uses UTF-8/surrogateescape */ if (preconfig->utf8_mode) { if (config->stdio_encoding == NULL) { - err = _PyCoreConfig_SetString(&config->stdio_encoding, L"utf-8"); + err = _PyCoreConfig_SetString(config, &config->stdio_encoding, L"utf-8"); if (_Py_INIT_FAILED(err)) { return err; } } if (config->stdio_errors == NULL) { - err = _PyCoreConfig_SetString(&config->stdio_errors, + err = _PyCoreConfig_SetString(config, &config->stdio_errors, L"surrogateescape"); if (_Py_INIT_FAILED(err)) { return err; @@ -1513,7 +1529,7 @@ config_init_stdio_encoding(_PyCoreConfig *config, /* Choose the default error handler based on the current locale. */ if (config->stdio_encoding == NULL) { - err = config_get_locale_encoding(&config->stdio_encoding); + err = config_get_locale_encoding(config, &config->stdio_encoding); if (_Py_INIT_FAILED(err)) { return err; } @@ -1522,7 +1538,7 @@ config_init_stdio_encoding(_PyCoreConfig *config, const wchar_t *errors = config_get_stdio_errors(config); assert(errors != NULL); - err = _PyCoreConfig_SetString(&config->stdio_errors, errors); + err = _PyCoreConfig_SetString(config, &config->stdio_errors, errors); if (_Py_INIT_FAILED(err)) { return err; } @@ -1539,34 +1555,35 @@ config_init_fs_encoding(_PyCoreConfig *config, const _PyPreConfig *preconfig) if (config->filesystem_encoding == NULL) { #ifdef _Py_FORCE_UTF8_FS_ENCODING - err = _PyCoreConfig_SetString(&config->filesystem_encoding, L"utf-8"); + err = _PyCoreConfig_SetString(config, &config->filesystem_encoding, L"utf-8"); #else #ifdef MS_WINDOWS if (preconfig->legacy_windows_fs_encoding) { /* Legacy Windows filesystem encoding: mbcs/replace */ - err = _PyCoreConfig_SetString(&config->filesystem_encoding, + err = _PyCoreConfig_SetString(config, &config->filesystem_encoding, L"mbcs"); } else #endif if (preconfig->utf8_mode) { - err = _PyCoreConfig_SetString(&config->filesystem_encoding, + err = _PyCoreConfig_SetString(config, &config->filesystem_encoding, L"utf-8"); } #ifndef MS_WINDOWS else if (_Py_GetForceASCII()) { - err = _PyCoreConfig_SetString(&config->filesystem_encoding, + err = _PyCoreConfig_SetString(config, &config->filesystem_encoding, L"ascii"); } #endif else { #ifdef MS_WINDOWS /* Windows defaults to utf-8/surrogatepass (PEP 529). */ - err = _PyCoreConfig_SetString(&config->filesystem_encoding, + err = _PyCoreConfig_SetString(config, &config->filesystem_encoding, L"utf-8"); #else - err = config_get_locale_encoding(&config->filesystem_encoding); + err = config_get_locale_encoding(config, + &config->filesystem_encoding); #endif } #endif /* !_Py_FORCE_UTF8_FS_ENCODING */ @@ -1588,7 +1605,7 @@ config_init_fs_encoding(_PyCoreConfig *config, const _PyPreConfig *preconfig) #else errors = L"surrogateescape"; #endif - err = _PyCoreConfig_SetString(&config->filesystem_errors, errors); + err = _PyCoreConfig_SetString(config, &config->filesystem_errors, errors); if (_Py_INIT_FAILED(err)) { return err; } @@ -1681,7 +1698,7 @@ config_read(_PyCoreConfig *config) } if (config->check_hash_pycs_mode == NULL) { - err = _PyCoreConfig_SetString(&config->check_hash_pycs_mode, + err = _PyCoreConfig_SetString(config, &config->check_hash_pycs_mode, L"default"); if (_Py_INIT_FAILED(err)) { return err; @@ -1832,7 +1849,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyWstrList *warnoptions, || wcscmp(_PyOS_optarg, L"never") == 0 || wcscmp(_PyOS_optarg, L"default") == 0) { - err = _PyCoreConfig_SetString(&config->check_hash_pycs_mode, + err = _PyCoreConfig_SetString(config, &config->check_hash_pycs_mode, _PyOS_optarg); if (_Py_INIT_FAILED(err)) { return err; @@ -1966,7 +1983,7 @@ config_parse_cmdline(_PyCoreConfig *config, _PyWstrList *warnoptions, /* Get warning options from PYTHONWARNINGS environment variable. */ static _PyInitError -config_init_env_warnoptions(const _PyCoreConfig *config, _PyWstrList *warnoptions) +config_init_env_warnoptions(_PyCoreConfig *config, _PyWstrList *warnoptions) { _PyInitError err; /* CONFIG_GET_ENV_DUP requires dest to be initialized to NULL */ @@ -2136,8 +2153,7 @@ core_read_precmdline(_PyCoreConfig *config, _PyPreCmdline *precmdline) } _PyPreConfig preconfig; - _PyPreConfig_Init(&preconfig); - _PyPreConfig_Copy(&preconfig, &_PyRuntime.preconfig); + _PyPreConfig_InitFromPreConfig(&preconfig, &_PyRuntime.preconfig); _PyPreConfig_GetCoreConfig(&preconfig, config); @@ -2211,24 +2227,11 @@ done: _PyInitError _PyCoreConfig_SetPyArgv(_PyCoreConfig *config, const _PyArgv *args) { - if (args->use_bytes_argv) { - _PyInitError err; - - err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(err)) { - return err; - } - _PyRuntimeState *runtime = &_PyRuntime; - - /* do nothing if Python is already pre-initialized: - _PyCoreConfig_Write() will update _PyRuntime.preconfig later */ - if (!runtime->pre_initialized) { - err = _Py_PreInitializeFromCoreConfig(config, args); - if (_Py_INIT_FAILED(err)) { - return err; - } - } + _PyInitError err = _Py_PreInitializeFromCoreConfig(config, args); + if (_Py_INIT_FAILED(err)) { + return err; } + return _PyArgv_AsWstrList(args, &config->argv); } @@ -2236,7 +2239,7 @@ _PyCoreConfig_SetPyArgv(_PyCoreConfig *config, const _PyArgv *args) /* Set config.argv: decode argv using Py_DecodeLocale(). Pre-initialize Python if needed to ensure that encodings are properly configured. */ _PyInitError -_PyCoreConfig_SetArgv(_PyCoreConfig *config, Py_ssize_t argc, char **argv) +_PyCoreConfig_SetArgv(_PyCoreConfig *config, Py_ssize_t argc, char * const *argv) { _PyArgv args = { .argc = argc, @@ -2248,7 +2251,7 @@ _PyCoreConfig_SetArgv(_PyCoreConfig *config, Py_ssize_t argc, char **argv) _PyInitError -_PyCoreConfig_SetWideArgv(_PyCoreConfig *config, Py_ssize_t argc, wchar_t **argv) +_PyCoreConfig_SetWideArgv(_PyCoreConfig *config, Py_ssize_t argc, wchar_t * const *argv) { _PyArgv args = { .argc = argc, diff --git a/Python/preconfig.c b/Python/preconfig.c index 0f4bd8ece53..71a6ee6c072 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -95,7 +95,7 @@ _PyArgv_AsWstrList(const _PyArgv *args, _PyWstrList *list) } else { wargv.length = args->argc; - wargv.items = args->wchar_argv; + wargv.items = (wchar_t **)args->wchar_argv; if (_PyWstrList_Copy(list, &wargv) < 0) { return _Py_INIT_NO_MEMORY(); } @@ -217,16 +217,15 @@ precmdline_parse_cmdline(_PyPreCmdline *cmdline) _PyInitError -_PyPreCmdline_Read(_PyPreCmdline *cmdline, - const _PyPreConfig *preconfig) +_PyPreCmdline_Read(_PyPreCmdline *cmdline, const _PyPreConfig *preconfig) { - if (preconfig) { - _PyPreCmdline_GetPreConfig(cmdline, preconfig); - } + _PyPreCmdline_GetPreConfig(cmdline, preconfig); - _PyInitError err = precmdline_parse_cmdline(cmdline); - if (_Py_INIT_FAILED(err)) { - return err; + if (preconfig->parse_argv) { + _PyInitError err = precmdline_parse_cmdline(cmdline); + if (_Py_INIT_FAILED(err)) { + return err; + } } /* isolated, use_environment */ @@ -268,6 +267,7 @@ _PyPreConfig_Init(_PyPreConfig *config) memset(config, 0, sizeof(*config)); config->_config_version = _Py_CONFIG_VERSION; + config->parse_argv = 0; config->isolated = -1; config->use_environment = -1; config->configure_locale = 1; @@ -285,6 +285,7 @@ _PyPreConfig_InitPythonConfig(_PyPreConfig *config) { _PyPreConfig_Init(config); + config->parse_argv = 1; /* Set to -1 to enable C locale coercion (PEP 538) and UTF-8 Mode (PEP 540) depending on the LC_CTYPE locale, PYTHONUTF8 and PYTHONCOERCECLOCALE environment variables. */ @@ -310,11 +311,41 @@ _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config) } +void +_PyPreConfig_InitFromPreConfig(_PyPreConfig *config, + const _PyPreConfig *config2) +{ + _PyPreConfig_Init(config); + _PyPreConfig_Copy(config, config2); +} + + +void +_PyPreConfig_InitFromCoreConfig(_PyPreConfig *config, + const _PyCoreConfig *coreconfig) +{ + _PyCoreConfigInitEnum config_init = (_PyCoreConfigInitEnum)coreconfig->_config_init; + switch (config_init) { + case _PyCoreConfig_INIT_PYTHON: + _PyPreConfig_InitPythonConfig(config); + break; + case _PyCoreConfig_INIT_ISOLATED: + _PyPreConfig_InitIsolatedConfig(config); + break; + case _PyCoreConfig_INIT: + default: + _PyPreConfig_Init(config); + } + _PyPreConfig_GetCoreConfig(config, coreconfig); +} + + void _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) { #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR + COPY_ATTR(parse_argv); COPY_ATTR(isolated); COPY_ATTR(use_environment); COPY_ATTR(configure_locale); @@ -341,27 +372,20 @@ _PyPreConfig_AsDict(const _PyPreConfig *config) return NULL; } -#define SET_ITEM(KEY, EXPR) \ +#define SET_ITEM_INT(ATTR) \ do { \ - PyObject *obj = (EXPR); \ + PyObject *obj = PyLong_FromLong(config->ATTR); \ if (obj == NULL) { \ goto fail; \ } \ - int res = PyDict_SetItemString(dict, (KEY), obj); \ + int res = PyDict_SetItemString(dict, #ATTR, obj); \ Py_DECREF(obj); \ if (res < 0) { \ goto fail; \ } \ } while (0) -#define SET_ITEM_INT(ATTR) \ - SET_ITEM(#ATTR, PyLong_FromLong(config->ATTR)) -#define FROM_STRING(STR) \ - ((STR != NULL) ? \ - PyUnicode_FromString(STR) \ - : (Py_INCREF(Py_None), Py_None)) -#define SET_ITEM_STR(ATTR) \ - SET_ITEM(#ATTR, FROM_STRING(config->ATTR)) + SET_ITEM_INT(parse_argv); SET_ITEM_INT(isolated); SET_ITEM_INT(use_environment); SET_ITEM_INT(configure_locale); @@ -379,10 +403,7 @@ fail: Py_DECREF(dict); return NULL; -#undef FROM_STRING -#undef SET_ITEM #undef SET_ITEM_INT -#undef SET_ITEM_STR } @@ -395,6 +416,7 @@ _PyPreConfig_GetCoreConfig(_PyPreConfig *config, config->ATTR = core_config->ATTR; \ } + COPY_ATTR(parse_argv); COPY_ATTR(isolated); COPY_ATTR(use_environment); COPY_ATTR(dev_mode); @@ -662,9 +684,11 @@ preconfig_init_allocator(_PyPreConfig *config) allocators to "malloc" (and not to "debug"). */ const char *envvar = _Py_GetEnv(config->use_environment, "PYTHONMALLOC"); if (envvar) { - if (_PyMem_GetAllocatorName(envvar, &config->allocator) < 0) { + PyMemAllocatorName name; + if (_PyMem_GetAllocatorName(envvar, &name) < 0) { return _Py_INIT_ERR("PYTHONMALLOC: unknown allocator"); } + config->allocator = (int)name; } } @@ -751,8 +775,7 @@ _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args) /* Save the config to be able to restore it if encodings change */ _PyPreConfig save_config; - _PyPreConfig_Init(&save_config); - _PyPreConfig_Copy(&save_config, config); + _PyPreConfig_InitFromPreConfig(&save_config, config); /* Set LC_CTYPE to the user preferred locale */ if (config->configure_locale) { @@ -879,8 +902,9 @@ _PyPreConfig_Write(const _PyPreConfig *config) return _Py_INIT_OK(); } - if (config->allocator != PYMEM_ALLOCATOR_NOT_SET) { - if (_PyMem_SetupAllocators(config->allocator) < 0) { + PyMemAllocatorName name = (PyMemAllocatorName)config->allocator; + if (name != PYMEM_ALLOCATOR_NOT_SET) { + if (_PyMem_SetupAllocators(name) < 0) { return _Py_INIT_ERR("Unknown PYTHONMALLOC allocator"); } } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 0781dc8046b..01f725f8f45 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -691,6 +691,10 @@ _Py_PreInitializeFromPyArgv(const _PyPreConfig *src_config, const _PyArgv *args) { _PyInitError err; + if (src_config == NULL) { + return _Py_INIT_ERR("preinitialization config is NULL"); + } + err = _PyRuntime_Initialize(); if (_Py_INIT_FAILED(err)) { return err; @@ -703,11 +707,7 @@ _Py_PreInitializeFromPyArgv(const _PyPreConfig *src_config, const _PyArgv *args) } _PyPreConfig config; - _PyPreConfig_Init(&config); - - if (src_config) { - _PyPreConfig_Copy(&config, src_config); - } + _PyPreConfig_InitFromPreConfig(&config, src_config); err = _PyPreConfig_Read(&config, args); if (_Py_INIT_FAILED(err)) { @@ -751,21 +751,34 @@ _PyInitError _Py_PreInitializeFromCoreConfig(const _PyCoreConfig *coreconfig, const _PyArgv *args) { - _PyPreConfig config; - _PyPreConfig_Init(&config); - if (coreconfig != NULL) { - _PyPreConfig_GetCoreConfig(&config, coreconfig); + assert(coreconfig != NULL); + + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + return err; + } + _PyRuntimeState *runtime = &_PyRuntime; + + if (runtime->pre_initialized) { + /* Already initialized: do nothing */ + return _Py_INIT_OK(); } - if (args == NULL && coreconfig != NULL && coreconfig->parse_argv) { + _PyPreConfig preconfig; + _PyPreConfig_InitFromCoreConfig(&preconfig, coreconfig); + + if (!coreconfig->parse_argv) { + return _Py_PreInitialize(&preconfig); + } + else if (args == NULL) { _PyArgv config_args = { .use_bytes_argv = 0, .argc = coreconfig->argv.length, .wchar_argv = coreconfig->argv.items}; - return _Py_PreInitializeFromPyArgv(&config, &config_args); + return _Py_PreInitializeFromPyArgv(&preconfig, &config_args); } else { - return _Py_PreInitializeFromPyArgv(&config, args); + return _Py_PreInitializeFromPyArgv(&preconfig, args); } } @@ -777,13 +790,11 @@ pyinit_coreconfig(_PyRuntimeState *runtime, const _PyArgv *args, PyInterpreterState **interp_p) { - _PyInitError err; + assert(src_config != NULL); - if (src_config) { - err = _PyCoreConfig_Copy(config, src_config); - if (_Py_INIT_FAILED(err)) { - return err; - } + _PyInitError err = _PyCoreConfig_Copy(config, src_config); + if (_Py_INIT_FAILED(err)) { + return err; } if (args) { @@ -995,6 +1006,10 @@ _Py_InitializeMain(void) static _PyInitError init_python(const _PyCoreConfig *config, const _PyArgv *args) { + if (config == NULL) { + return _Py_INIT_ERR("initialization config is NULL"); + } + _PyInitError err; err = _PyRuntime_Initialize(); @@ -1022,7 +1037,8 @@ init_python(const _PyCoreConfig *config, const _PyArgv *args) _PyInitError -_Py_InitializeFromArgs(const _PyCoreConfig *config, Py_ssize_t argc, char **argv) +_Py_InitializeFromArgs(const _PyCoreConfig *config, + Py_ssize_t argc, char * const *argv) { _PyArgv args = {.use_bytes_argv = 1, .argc = argc, .bytes_argv = argv}; return init_python(config, &args); @@ -1030,7 +1046,8 @@ _Py_InitializeFromArgs(const _PyCoreConfig *config, Py_ssize_t argc, char **argv _PyInitError -_Py_InitializeFromWideArgs(const _PyCoreConfig *config, Py_ssize_t argc, wchar_t **argv) +_Py_InitializeFromWideArgs(const _PyCoreConfig *config, + Py_ssize_t argc, wchar_t * const *argv) { _PyArgv args = {.use_bytes_argv = 0, .argc = argc, .wchar_argv = argv}; return init_python(config, &args);