From 1075d1684ab84dc7c28d93cfb46e95e70d3b6d3b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 25 Mar 2019 23:19:57 +0100 Subject: [PATCH] bpo-36301: Add _Py_GetConfigsAsDict() function (GH-12540) * Add _Py_GetConfigsAsDict() function to get all configurations as a dict. * dump_config() of _testembed.c now dumps preconfig as a separated key: call _Py_GetConfigsAsDict(). * Make _PyMainInterpreterConfig_AsDict() private. --- Include/cpython/coreconfig.h | 3 +- Include/cpython/pylifecycle.h | 3 - Include/internal/pycore_coreconfig.h | 8 +- Lib/test/pythoninfo.py | 13 ++- Lib/test/test_embed.py | 114 ++++++++++++++++++--------- Modules/_testcapimodule.c | 26 +----- Programs/_testembed.c | 57 +++----------- Python/coreconfig.c | 73 +++++++++++++++-- Python/preconfig.c | 16 +++- 9 files changed, 179 insertions(+), 134 deletions(-) diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index bb086cbd125..621a09fa79e 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -400,8 +400,7 @@ typedef struct { /* --- Function used for testing ---------------------------------- */ -PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); -PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); +PyAPI_FUNC(PyObject*) _Py_GetConfigsAsDict(void); #ifdef __cplusplus } diff --git a/Include/cpython/pylifecycle.h b/Include/cpython/pylifecycle.h index a226de8446c..5f3a522a600 100644 --- a/Include/cpython/pylifecycle.h +++ b/Include/cpython/pylifecycle.h @@ -33,9 +33,6 @@ PyAPI_FUNC(void) _PyMainInterpreterConfig_Clear(_PyMainInterpreterConfig *); PyAPI_FUNC(int) _PyMainInterpreterConfig_Copy( _PyMainInterpreterConfig *config, const _PyMainInterpreterConfig *config2); -/* Used by _testcapi.get_main_config() */ -PyAPI_FUNC(PyObject*) _PyMainInterpreterConfig_AsDict( - const _PyMainInterpreterConfig *config); PyAPI_FUNC(_PyInitError) _Py_InitializeMainInterpreter( PyInterpreterState *interp, diff --git a/Include/internal/pycore_coreconfig.h b/Include/internal/pycore_coreconfig.h index d44172e0dd8..3c840101b84 100644 --- a/Include/internal/pycore_coreconfig.h +++ b/Include/internal/pycore_coreconfig.h @@ -90,8 +90,7 @@ PyAPI_FUNC(void) _Py_get_env_flag(_PyPreConfig *config, PyAPI_FUNC(_PyInitError) _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args, const _PyCoreConfig *coreconfig); -PyAPI_FUNC(int) _PyPreConfig_AsDict(const _PyPreConfig *config, - PyObject *dict); +PyAPI_FUNC(PyObject*) _PyPreConfig_AsDict(const _PyPreConfig *config); PyAPI_FUNC(_PyInitError) _PyPreConfig_ReadFromArgv(_PyPreConfig *config, const _PyArgv *args); PyAPI_FUNC(_PyInitError) _PyPreConfig_Write(_PyPreConfig *config); @@ -121,6 +120,11 @@ PyAPI_FUNC(_PyInitError) _PyCoreConfig_ReadFromArgv(_PyCoreConfig *config, const _PyArgv *args); PyAPI_FUNC(_PyInitError) _PyCoreConfig_Write(const _PyCoreConfig *config); +/* --- _PyMainInterpreterConfig ----------------------------------- */ + +PyAPI_FUNC(PyObject*) _PyMainInterpreterConfig_AsDict( + const _PyMainInterpreterConfig *config); + #ifdef __cplusplus } #endif diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 07f3cb3bea4..79f7e82e000 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -598,18 +598,15 @@ def collect_get_config(info_add): # Dump global configuration variables, _PyCoreConfig # and _PyMainInterpreterConfig try: - from _testcapi import get_global_config, get_core_config, get_main_config + from _testcapi import get_configs except ImportError: return - for prefix, get_config_func in ( - ('global_config', get_global_config), - ('core_config', get_core_config), - ('main_config', get_main_config), - ): - config = get_config_func() + all_configs = get_configs() + for config_type in sorted(all_configs): + config = all_configs[config_type] for key in sorted(config): - info_add('%s[%s]' % (prefix, key), repr(config[key])) + info_add('%s[%s]' % (config_type, key), repr(config[key])) def collect_subprocess(info_add): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 374346e3cc8..6e145a5aa13 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -268,13 +268,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ) # Mark config which should be get by get_default_config() GET_DEFAULT_CONFIG = object() + DEFAULT_PRE_CONFIG = { + 'allocator': None, + 'coerce_c_locale': 0, + 'coerce_c_locale_warn': 0, + 'dev_mode': 0, + 'isolated': 0, + 'use_environment': 1, + 'utf8_mode': 0, + } DEFAULT_CORE_CONFIG = { 'install_signal_handlers': 1, - 'use_environment': 1, 'use_hash_seed': 0, 'hash_seed': 0, - 'allocator': None, - 'dev_mode': 0, 'faulthandler': 0, 'tracemalloc': 0, 'import_time': 0, @@ -286,10 +292,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'filesystem_encoding': GET_DEFAULT_CONFIG, 'filesystem_errors': GET_DEFAULT_CONFIG, - 'utf8_mode': 0, - 'coerce_c_locale': 0, - 'coerce_c_locale_warn': 0, - 'pycache_prefix': None, 'program_name': './_testembed', 'argv': [""], @@ -306,7 +308,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, - 'isolated': 0, 'site_import': 1, 'bytes_warning': 0, 'inspect': 0, @@ -332,8 +333,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_frozen': 0, } if MS_WINDOWS: - DEFAULT_CORE_CONFIG.update({ + DEFAULT_PRE_CONFIG.update({ 'legacy_windows_fs_encoding': 0, + }) + DEFAULT_CORE_CONFIG.update({ 'legacy_windows_stdio': 0, }) @@ -359,6 +362,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'Py_HashRandomizationFlag': 1, '_Py_HasFileSystemDefaultEncodeErrors': 0, } + COPY_GLOBAL_PRE_CONFIG = [ + ('Py_IgnoreEnvironmentFlag', 'use_environment', True), + ('Py_IsolatedFlag', 'isolated'), + ('Py_UTF8Mode', 'utf8_mode'), + ] COPY_GLOBAL_CONFIG = [ # Copy core config to global config for expected values # True means that the core config value is inverted (0 => 1 and 1 => 0) @@ -368,21 +376,20 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), ('Py_FrozenFlag', '_frozen'), - ('Py_IgnoreEnvironmentFlag', 'use_environment', True), ('Py_InspectFlag', 'inspect'), ('Py_InteractiveFlag', 'interactive'), - ('Py_IsolatedFlag', 'isolated'), ('Py_NoSiteFlag', 'site_import', True), ('Py_NoUserSiteDirectory', 'user_site_directory', True), ('Py_OptimizeFlag', 'optimization_level'), ('Py_QuietFlag', 'quiet'), - ('Py_UTF8Mode', 'utf8_mode'), ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), ('Py_VerboseFlag', 'verbose'), ] if MS_WINDOWS: - COPY_GLOBAL_CONFIG.extend(( + COPY_GLOBAL_PRE_CONFIG.extend(( ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), + )) + COPY_GLOBAL_CONFIG.extend(( ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), )) @@ -408,7 +415,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected['xoptions'] = self.main_xoptions(core_config['xoptions']) self.assertEqual(main_config, expected) - def get_expected_config(self, expected, env): + def get_expected_config(self, expected, expected_preconfig, env): expected = dict(self.DEFAULT_CORE_CONFIG, **expected) code = textwrap.dedent(''' @@ -436,7 +443,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # when test_embed is run from a venv (bpo-35313) args = (sys.executable, '-S', '-c', code) env = dict(env) - if not expected['isolated']: + if not expected_preconfig['isolated']: env['PYTHONCOERCECLOCALE'] = '0' env['PYTHONUTF8'] = '0' proc = subprocess.run(args, env=env, @@ -453,6 +460,11 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected[key] = config[key] return expected + def check_pre_config(self, config, expected): + pre_config = dict(config['pre_config']) + core_config = dict(config['core_config']) + self.assertEqual(pre_config, expected) + def check_core_config(self, config, expected): core_config = dict(config['core_config']) for key in self.UNTESTED_CORE_CONFIG: @@ -460,6 +472,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.assertEqual(core_config, expected) def check_global_config(self, config): + pre_config = config['pre_config'] core_config = config['core_config'] expected = dict(self.DEFAULT_GLOBAL_CONFIG) @@ -470,10 +483,17 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): else: global_key, core_key = item expected[global_key] = core_config[core_key] + for item in self.COPY_GLOBAL_PRE_CONFIG: + if len(item) == 3: + global_key, core_key, opposite = item + expected[global_key] = 0 if pre_config[core_key] else 1 + else: + global_key, core_key = item + expected[global_key] = pre_config[core_key] self.assertEqual(config['global_config'], expected) - def check_config(self, testname, expected): + def check_config(self, testname, expected_config, expected_preconfig): env = dict(os.environ) # Remove PYTHON* environment variables to get deterministic environment for key in list(env): @@ -488,15 +508,21 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # Ignore err config = json.loads(out) - expected = self.get_expected_config(expected, env) - self.check_core_config(config, expected) + expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig) + expected_config = self.get_expected_config(expected_config, expected_preconfig, env) + + self.check_core_config(config, expected_config) + self.check_pre_config(config, expected_preconfig) self.check_main_config(config) self.check_global_config(config) def test_init_default_config(self): - self.check_config("init_default_config", {}) + self.check_config("init_default_config", {}, {}) def test_init_global_config(self): + preconfig = { + 'utf8_mode': 1, + } config = { 'program_name': './globalvar', 'site_import': 0, @@ -509,7 +535,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'quiet': 1, 'buffered_stdio': 0, - 'utf8_mode': 1, 'stdio_encoding': 'utf-8', 'stdio_errors': 'surrogateescape', 'filesystem_encoding': 'utf-8', @@ -517,21 +542,23 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'user_site_directory': 0, '_frozen': 1, } - self.check_config("init_global_config", config) + self.check_config("init_global_config", config, preconfig) def test_init_from_config(self): + preconfig = { + 'allocator': 'malloc', + 'utf8_mode': 1, + } config = { 'install_signal_handlers': 0, 'use_hash_seed': 1, 'hash_seed': 123, - 'allocator': 'malloc', 'tracemalloc': 2, 'import_time': 1, 'show_ref_count': 1, 'show_alloc_count': 1, 'malloc_stats': 1, - 'utf8_mode': 1, 'stdio_encoding': 'iso8859-1', 'stdio_errors': 'replace', 'filesystem_encoding': 'utf-8', @@ -559,16 +586,18 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '_check_hash_pycs_mode': 'always', '_frozen': 1, } - self.check_config("init_from_config", config) + self.check_config("init_from_config", config, preconfig) + INIT_ENV_PRECONFIG = { + 'allocator': 'malloc', + 'utf8_mode': 1, + } INIT_ENV_CONFIG = { 'use_hash_seed': 1, 'hash_seed': 42, - 'allocator': 'malloc', 'tracemalloc': 2, 'import_time': 1, 'malloc_stats': 1, - 'utf8_mode': 1, 'filesystem_encoding': 'utf-8', 'filesystem_errors': UTF8_MODE_ERRORS, 'inspect': 1, @@ -584,35 +613,42 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): } def test_init_env(self): - self.check_config("init_env", self.INIT_ENV_CONFIG) + self.check_config("init_env", self.INIT_ENV_CONFIG, self.INIT_ENV_PRECONFIG) def test_init_env_dev_mode(self): - config = dict(self.INIT_ENV_CONFIG, + preconfig = dict(self.INIT_ENV_PRECONFIG, allocator='debug', dev_mode=1) - self.check_config("init_env_dev_mode", config) + config = dict(self.INIT_ENV_CONFIG, + dev_mode=1) + self.check_config("init_env_dev_mode", config, preconfig) def test_init_env_dev_mode(self): - config = dict(self.INIT_ENV_CONFIG, - allocator='malloc', - dev_mode=1) - self.check_config("init_env_dev_mode_alloc", config) + preconfig = dict(self.INIT_ENV_PRECONFIG, + allocator='malloc', + dev_mode=1) + config = dict(self.INIT_ENV_CONFIG) + self.check_config("init_env_dev_mode_alloc", config, preconfig) def test_init_dev_mode(self): - config = { - 'dev_mode': 1, - 'faulthandler': 1, + preconfig = { 'allocator': 'debug', + 'dev_mode': 1, } - self.check_config("init_dev_mode", config) + config = { + 'faulthandler': 1, + } + self.check_config("init_dev_mode", config, preconfig) def test_init_isolated(self): - config = { + preconfig = { 'isolated': 1, 'use_environment': 0, + } + config = { 'user_site_directory': 0, } - self.check_config("init_isolated", config) + self.check_config("init_isolated", config, preconfig) if __name__ == "__main__": diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 350ef771630..c82ba0cfd0d 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4675,27 +4675,9 @@ decode_locale_ex(PyObject *self, PyObject *args) static PyObject * -get_global_config(PyObject *self, PyObject *Py_UNUSED(args)) +get_configs(PyObject *self, PyObject *Py_UNUSED(args)) { - return _Py_GetGlobalVariablesAsDict(); -} - - -static PyObject * -get_core_config(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyInterpreterState *interp = _PyInterpreterState_Get(); - const _PyCoreConfig *config = _PyInterpreterState_GetCoreConfig(interp); - return _PyCoreConfig_AsDict(config); -} - - -static PyObject * -get_main_config(PyObject *self, PyObject *Py_UNUSED(args)) -{ - PyInterpreterState *interp = _PyInterpreterState_Get(); - const _PyMainInterpreterConfig *config = _PyInterpreterState_GetMainConfig(interp); - return _PyMainInterpreterConfig_AsDict(config); + return _Py_GetConfigsAsDict(); } @@ -4942,9 +4924,7 @@ static PyMethodDef TestMethods[] = { {"bad_get", (PyCFunction)(void(*)(void))bad_get, METH_FASTCALL}, {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, - {"get_global_config", get_global_config, METH_NOARGS}, - {"get_core_config", get_core_config, METH_NOARGS}, - {"get_main_config", get_main_config, METH_NOARGS}, + {"get_configs", get_configs, METH_NOARGS}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 7c143f1ef38..ab5802da363 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -301,64 +301,29 @@ static int test_initialize_pymain(void) static int dump_config_impl(void) { - PyObject *config = NULL; - PyObject *dict = NULL; - - config = PyDict_New(); + PyObject *config = _Py_GetConfigsAsDict(); if (config == NULL) { - goto error; + return -1; } - /* global config */ - dict = _Py_GetGlobalVariablesAsDict(); - if (dict == NULL) { - goto error; - } - if (PyDict_SetItemString(config, "global_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - - /* core config */ - PyInterpreterState *interp = _PyInterpreterState_Get(); - const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp); - dict = _PyCoreConfig_AsDict(core_config); - if (dict == NULL) { - goto error; - } - if (PyDict_SetItemString(config, "core_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - - /* main config */ - const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp); - dict = _PyMainInterpreterConfig_AsDict(main_config); - if (dict == NULL) { - goto error; - } - if (PyDict_SetItemString(config, "main_config", dict) < 0) { - goto error; - } - Py_CLEAR(dict); - + PyObject *res; PyObject *json = PyImport_ImportModule("json"); - PyObject *res = PyObject_CallMethod(json, "dumps", "O", config); - Py_DECREF(json); + if (json) { + res = PyObject_CallMethod(json, "dumps", "O", config); + Py_DECREF(json); + } + else { + res = NULL; + } Py_CLEAR(config); if (res == NULL) { - goto error; + return -1; } PySys_FormatStdout("%S\n", res); Py_DECREF(res); return 0; - -error: - Py_XDECREF(config); - Py_XDECREF(dict); - return -1; } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index ba5abb6ca40..a434b258f2a 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -131,7 +131,7 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif -PyObject * +static PyObject * _Py_GetGlobalVariablesAsDict(void) { PyObject *dict, *obj; @@ -1563,7 +1563,7 @@ _PyCoreConfig_Write(const _PyCoreConfig *config) } -PyObject * +static PyObject * _PyCoreConfig_AsDict(const _PyCoreConfig *config) { PyObject *dict; @@ -1573,11 +1573,6 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) return NULL; } - if (_PyPreConfig_AsDict(&config->preconfig, dict) < 0) { - Py_DECREF(dict); - return NULL; - } - #define SET_ITEM(KEY, EXPR) \ do { \ PyObject *obj = (EXPR); \ @@ -2158,3 +2153,67 @@ done: cmdline_clear(&cmdline); return err; } + + +PyObject* +_Py_GetConfigsAsDict(void) +{ + PyObject *config = NULL; + PyObject *dict = NULL; + + config = PyDict_New(); + if (config == NULL) { + goto error; + } + + /* global config */ + dict = _Py_GetGlobalVariablesAsDict(); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "global_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* pre config */ + PyInterpreterState *interp = _PyInterpreterState_Get(); + const _PyCoreConfig *core_config = _PyInterpreterState_GetCoreConfig(interp); + const _PyPreConfig *pre_config = &core_config->preconfig; + dict = _PyPreConfig_AsDict(pre_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "pre_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* core config */ + dict = _PyCoreConfig_AsDict(core_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "core_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* main config */ + const _PyMainInterpreterConfig *main_config = _PyInterpreterState_GetMainConfig(interp); + dict = _PyMainInterpreterConfig_AsDict(main_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "main_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + return config; + +error: + Py_XDECREF(config); + Py_XDECREF(dict); + return NULL; +} diff --git a/Python/preconfig.c b/Python/preconfig.c index ac87a7a3c7e..c65ee28f73a 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -574,9 +574,16 @@ _PyPreCmdline_SetPreConfig(const _PyPreCmdline *cmdline, _PyPreConfig *config) } -int -_PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) +PyObject* +_PyPreConfig_AsDict(const _PyPreConfig *config) { + PyObject *dict; + + dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + #define SET_ITEM(KEY, EXPR) \ do { \ PyObject *obj = (EXPR); \ @@ -608,10 +615,11 @@ _PyPreConfig_AsDict(const _PyPreConfig *config, PyObject *dict) #endif SET_ITEM_INT(dev_mode); SET_ITEM_STR(allocator); - return 0; + return dict; fail: - return -1; + Py_DECREF(dict); + return NULL; #undef FROM_STRING #undef SET_ITEM