From 7ddd56f4d835c6107b20a0b4233185bb59270142 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 14 Nov 2018 00:24:28 +0100 Subject: [PATCH] bpo-35233: Rewrite test_embed.InitConfigTests (GH-10524) * Fix _PyCoreConfig_SetGlobalConfig(): set also Py_FrozenFlag * Fix _PyCoreConfig_AsDict(): export also xoptions * Add _Py_GetGlobalVariablesAsDict() and _testcapi.get_global_config() * test.pythoninfo: dump also global configuration variables * _testembed now serializes global, core and main configurations using JSON to reuse _Py_GetGlobalVariablesAsDict(), _PyCoreConfig_AsDict() and _PyMainInterpreterConfig_AsDict(), rather than duplicating code. * test_embed.InitConfigTests now test much more configuration variables --- Include/coreconfig.h | 1 + Lib/test/pythoninfo.py | 25 +++--- Lib/test/test_embed.py | 126 ++++++++++++++++++--------- Modules/_testcapimodule.c | 16 +++- Programs/_testembed.c | 175 ++++++++++++-------------------------- Python/coreconfig.c | 75 ++++++++++++++++ 6 files changed, 240 insertions(+), 178 deletions(-) diff --git a/Include/coreconfig.h b/Include/coreconfig.h index a7d3983d4d1..f71a364cc2c 100644 --- a/Include/coreconfig.h +++ b/Include/coreconfig.h @@ -361,6 +361,7 @@ PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( /* Used by _testcapi.get_coreconfig() */ PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); +PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); #endif #ifdef __cplusplus diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py index 2b5d6e20a25..9257fdf332a 100644 --- a/Lib/test/pythoninfo.py +++ b/Lib/test/pythoninfo.py @@ -535,24 +535,21 @@ def collect_gdbm(info_add): def collect_get_config(info_add): - # Dump _PyCoreConfig and _PyMainInterpreterConfig + # Dump global configuration variables, _PyCoreConfig + # and _PyMainInterpreterConfig try: - from _testcapi import get_coreconfig + from _testcapi import get_global_config, get_core_config, get_main_config except ImportError: - pass - else: - config = get_coreconfig() - for key in sorted(config): - info_add('core_config[%s]' % key, repr(config[key])) + return - try: - from _testcapi import get_mainconfig - except ImportError: - pass - else: - config = get_mainconfig() + for prefix, get_config_func in ( + ('global_config', get_global_config), + ('core_config', get_core_config), + ('main_config', get_main_config), + ): + config = get_config_func() for key in sorted(config): - info_add('main_config[%s]' % key, repr(config[key])) + info_add('%s[%s]' % (prefix, key), repr(config[key])) def collect_info(info): diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index cd10376c249..03a11842fa6 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -3,6 +3,7 @@ from test import support import unittest from collections import namedtuple +import json import os import platform import re @@ -10,9 +11,6 @@ import subprocess import sys -# AIX libc prints an empty string as '' rather than the string '(null)' -NULL_STR = '' if platform.system() == 'AIX' else '(null)' - class EmbeddingTestsMixin: def setUp(self): here = os.path.abspath(__file__) @@ -255,16 +253,32 @@ class EmbeddingTests(EmbeddingTestsMixin, unittest.TestCase): class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 - CORE_CONFIG_REGEX = re.compile(r"^core_config\[([^]]*)\] = (.*)$") - MAIN_CONFIG_REGEX = re.compile(r"^main_config\[([^]]*)\] = (.*)$") UTF8_MODE_ERRORS = ('surrogatepass' if sys.platform == 'win32' else 'surrogateescape') + # FIXME: untested core configuration variables + UNTESTED_CORE_CONFIG = ( + 'base_exec_prefix', + 'base_prefix', + 'dll_path', + 'exec_prefix', + 'executable', + 'home', + 'legacy_windows_fs_encoding', + 'legacy_windows_stdio', + 'module_search_path_env', + 'module_search_paths', + 'prefix', + ) + # FIXME: untested main configuration variables + UNTESTED_MAIN_CONFIG = ( + 'module_search_path', + ) DEFAULT_CORE_CONFIG = { 'install_signal_handlers': 1, 'use_environment': 1, 'use_hash_seed': 0, 'hash_seed': 0, - 'allocator': NULL_STR, + 'allocator': None, 'dev_mode': 0, 'faulthandler': 0, 'tracemalloc': 0, @@ -282,11 +296,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'coerce_c_locale': 0, 'coerce_c_locale_warn': 0, - 'pycache_prefix': NULL_STR, + 'pycache_prefix': None, 'program_name': './_testembed', - 'argc': 0, - 'argv': '[]', - 'program': NULL_STR, + 'argv': [], + 'program': None, + + 'xoptions': [], + 'warnoptions': [], 'isolated': 0, 'site_import': 1, @@ -363,46 +379,76 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): expected['filesystem_encoding'] = res[0] if expected['filesystem_errors'] is None: expected['filesystem_errors'] = res[1] - for key, value in expected.items(): - expected[key] = str(value) out, err = self.run_embedded_interpreter(testname, env=env) # Ignore err - core_config = {} - main_config = {} - for line in out.splitlines(): - match = self.CORE_CONFIG_REGEX.match(line) - if match is not None: - key = match.group(1) - value = match.group(2) - core_config[key] = value - else: - match = self.MAIN_CONFIG_REGEX.match(line) - if match is None: - raise ValueError(f"failed to parse line {line!r}") - key = match.group(1) - value = match.group(2) - main_config[key] = value - self.assertEqual(core_config, expected) + config = json.loads(out) + core_config = config['core_config'] + executable = core_config['executable'] + main_config = config['main_config'] + + for key in self.UNTESTED_MAIN_CONFIG: + del main_config[key] - pycache_prefix = core_config['pycache_prefix'] - if pycache_prefix != NULL_STR: - pycache_prefix = repr(pycache_prefix) - else: - pycache_prefix = "NULL" expected_main = { 'install_signal_handlers': core_config['install_signal_handlers'], - 'argv': '[]', - 'prefix': repr(sys.prefix), - 'base_prefix': repr(sys.base_prefix), - 'base_exec_prefix': repr(sys.base_exec_prefix), - 'warnoptions': '[]', - 'xoptions': '{}', - 'pycache_prefix': pycache_prefix, + 'argv': [], + 'prefix': sys.prefix, + 'executable': core_config['executable'], + 'base_prefix': sys.base_prefix, + 'base_exec_prefix': sys.base_exec_prefix, + 'warnoptions': core_config['warnoptions'], + 'xoptions': {}, + 'pycache_prefix': core_config['pycache_prefix'], + 'exec_prefix': core_config['exec_prefix'], } self.assertEqual(main_config, expected_main) + + copy_global_config = [ + ('Py_BytesWarningFlag', 'bytes_warning'), + ('Py_DebugFlag', 'parser_debug'), + ('Py_DontWriteBytecodeFlag', 'write_bytecode', True), + ('Py_FileSystemDefaultEncodeErrors', 'filesystem_errors'), + ('Py_FileSystemDefaultEncoding', 'filesystem_encoding'), + ('Py_FrozenFlag', '_frozen'), + ('Py_IgnoreEnvironmentFlag', 'use_environment', True), + ('Py_InspectFlag', 'inspect'), + ('Py_InteractiveFlag', 'interactive'), + ('Py_IsolatedFlag', 'isolated'), + ('Py_NoSiteFlag', 'site_import', True), + ('Py_NoUserSiteDirectory', 'user_site_directory', True), + ('Py_OptimizeFlag', 'optimization_level'), + ('Py_QuietFlag', 'quiet'), + ('Py_UTF8Mode', 'utf8_mode'), + ('Py_UnbufferedStdioFlag', 'buffered_stdio', True), + ('Py_VerboseFlag', 'verbose'), + ] + if os.name == 'nt': + copy_global_config.extend(( + ('Py_LegacyWindowsFSEncodingFlag', 'legacy_windows_fs_encoding'), + ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), + )) + + expected_global = {} + for item in copy_global_config: + if len(item) == 3: + global_key, core_key, opposite = item + expected_global[global_key] = 0 if core_config[core_key] else 1 + else: + global_key, core_key = item + expected_global[global_key] = core_config[core_key] + + expected_global['Py_HasFileSystemDefaultEncoding'] = 0 + expected_global['_Py_HasFileSystemDefaultEncodeErrors'] = 0 + expected_global['Py_HashRandomizationFlag'] = 1 + self.assertEqual(config['global_config'], expected_global) + + for key in self.UNTESTED_CORE_CONFIG: + core_config.pop(key, None) + self.assertEqual(core_config, expected) + def test_init_default_config(self): self.check_config("init_default_config", {}) diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 205b668bdfc..56a08a418b3 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4694,7 +4694,14 @@ decode_locale_ex(PyObject *self, PyObject *args) static PyObject * -get_coreconfig(PyObject *self, PyObject *Py_UNUSED(args)) +get_global_config(PyObject *self, PyObject *Py_UNUSED(args)) +{ + return _Py_GetGlobalVariablesAsDict(); +} + + +static PyObject * +get_core_config(PyObject *self, PyObject *Py_UNUSED(args)) { PyInterpreterState *interp = _PyInterpreterState_Get(); const _PyCoreConfig *config = &interp->core_config; @@ -4703,7 +4710,7 @@ get_coreconfig(PyObject *self, PyObject *Py_UNUSED(args)) static PyObject * -get_mainconfig(PyObject *self, PyObject *Py_UNUSED(args)) +get_main_config(PyObject *self, PyObject *Py_UNUSED(args)) { PyInterpreterState *interp = _PyInterpreterState_Get(); const _PyMainInterpreterConfig *config = &interp->config; @@ -4956,8 +4963,9 @@ static PyMethodDef TestMethods[] = { {"bad_get", bad_get, METH_FASTCALL}, {"EncodeLocaleEx", encode_locale_ex, METH_VARARGS}, {"DecodeLocaleEx", decode_locale_ex, METH_VARARGS}, - {"get_coreconfig", get_coreconfig, METH_NOARGS}, - {"get_mainconfig", get_mainconfig, METH_NOARGS}, + {"get_global_config", get_global_config, METH_NOARGS}, + {"get_core_config", get_core_config, METH_NOARGS}, + {"get_main_config", get_main_config, METH_NOARGS}, #ifdef Py_REF_DEBUG {"negative_refcount", negative_refcount, METH_NOARGS}, #endif diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 27060718b49..12dc0f98be4 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -292,141 +292,76 @@ static int test_initialize_pymain(void) } -static void -dump_core_config(void) +static int +dump_config_impl(void) { -#define ASSERT_EQUAL(a, b) \ - if ((a) != (b)) { \ - printf("ERROR: %s != %s (%i != %i)\n", #a, #b, (a), (b)); \ - exit(1); \ - } -#define ASSERT_STR_EQUAL(a, b) \ - if ((a) == NULL || (b == NULL) || wcscmp((a), (b)) != 0) { \ - printf("ERROR: %s != %s ('%ls' != '%ls')\n", #a, #b, (a), (b)); \ - exit(1); \ + PyObject *config = NULL; + PyObject *dict = NULL; + + config = PyDict_New(); + if (config == NULL) { + goto error; } + /* global config */ + dict = _Py_GetGlobalVariablesAsDict(); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "global_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); + + /* core config */ PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyCoreConfig *config = &interp->core_config; - - printf("core_config[install_signal_handlers] = %i\n", config->install_signal_handlers); - - printf("core_config[use_environment] = %i\n", config->use_environment); - ASSERT_EQUAL(config->use_environment, !Py_IgnoreEnvironmentFlag); - - printf("core_config[use_hash_seed] = %i\n", config->use_hash_seed); - printf("core_config[hash_seed] = %lu\n", config->hash_seed); - - printf("core_config[allocator] = %s\n", config->allocator); - - printf("core_config[dev_mode] = %i\n", config->dev_mode); - printf("core_config[faulthandler] = %i\n", config->faulthandler); - printf("core_config[tracemalloc] = %i\n", config->tracemalloc); - printf("core_config[import_time] = %i\n", config->import_time); - printf("core_config[show_ref_count] = %i\n", config->show_ref_count); - printf("core_config[show_alloc_count] = %i\n", config->show_alloc_count); - printf("core_config[dump_refs] = %i\n", config->dump_refs); - printf("core_config[malloc_stats] = %i\n", config->malloc_stats); - - printf("core_config[filesystem_encoding] = %s\n", config->filesystem_encoding); - printf("core_config[filesystem_errors] = %s\n", config->filesystem_errors); - printf("core_config[coerce_c_locale] = %i\n", config->coerce_c_locale); - printf("core_config[coerce_c_locale_warn] = %i\n", config->coerce_c_locale_warn); - printf("core_config[utf8_mode] = %i\n", config->utf8_mode); - - printf("core_config[pycache_prefix] = %ls\n", config->pycache_prefix); - printf("core_config[program_name] = %ls\n", config->program_name); - ASSERT_STR_EQUAL(config->program_name, Py_GetProgramName()); - - printf("core_config[argc] = %i\n", config->argc); - printf("core_config[argv] = ["); - for (int i=0; i < config->argc; i++) { - if (i) { - printf(", "); - } - printf("\"%ls\"", config->argv[i]); + const _PyCoreConfig *core_config = &interp->core_config; + dict = _PyCoreConfig_AsDict(core_config); + if (dict == NULL) { + goto error; } - printf("]\n"); + if (PyDict_SetItemString(config, "core_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); - printf("core_config[program] = %ls\n", config->program); - /* FIXME: test xoptions */ - /* FIXME: test warnoptions */ - /* FIXME: test module_search_path_env */ - /* FIXME: test home */ - /* FIXME: test module_search_paths */ - /* FIXME: test executable */ - /* FIXME: test prefix */ - /* FIXME: test base_prefix */ - /* FIXME: test exec_prefix */ - /* FIXME: test base_exec_prefix */ - /* FIXME: test dll_path */ + /* main config */ + const _PyMainInterpreterConfig *main_config = &interp->config; + dict = _PyMainInterpreterConfig_AsDict(main_config); + if (dict == NULL) { + goto error; + } + if (PyDict_SetItemString(config, "main_config", dict) < 0) { + goto error; + } + Py_CLEAR(dict); - printf("core_config[isolated] = %i\n", config->isolated); - ASSERT_EQUAL(config->isolated, Py_IsolatedFlag); - printf("core_config[site_import] = %i\n", config->site_import); - printf("core_config[bytes_warning] = %i\n", config->bytes_warning); - printf("core_config[inspect] = %i\n", config->inspect); - printf("core_config[interactive] = %i\n", config->interactive); - printf("core_config[optimization_level] = %i\n", config->optimization_level); - printf("core_config[parser_debug] = %i\n", config->parser_debug); - printf("core_config[write_bytecode] = %i\n", config->write_bytecode); - printf("core_config[verbose] = %i\n", config->verbose); - ASSERT_EQUAL(config->verbose, Py_VerboseFlag); - printf("core_config[quiet] = %i\n", config->quiet); - printf("core_config[user_site_directory] = %i\n", config->user_site_directory); - printf("core_config[buffered_stdio] = %i\n", config->buffered_stdio); - ASSERT_EQUAL(config->buffered_stdio, !Py_UnbufferedStdioFlag); - printf("core_config[stdio_encoding] = %s\n", config->stdio_encoding); - printf("core_config[stdio_errors] = %s\n", config->stdio_errors); + PyObject *json = PyImport_ImportModule("json"); + PyObject *res = PyObject_CallMethod(json, "dumps", "O", config); + Py_DECREF(json); + Py_CLEAR(config); + if (res == NULL) { + goto error; + } - /* FIXME: test legacy_windows_fs_encoding */ - /* FIXME: test legacy_windows_stdio */ + PySys_FormatStdout("%S\n", res); + Py_DECREF(res); - printf("core_config[_install_importlib] = %i\n", config->_install_importlib); - printf("core_config[_check_hash_pycs_mode] = %s\n", config->_check_hash_pycs_mode); - printf("core_config[_frozen] = %i\n", config->_frozen); + return 0; -#undef ASSERT_EQUAL -#undef ASSERT_STR_EQUAL +error: + Py_XDECREF(config); + Py_XDECREF(dict); + return -1; } - -static void -dump_main_config(void) -{ - PyInterpreterState *interp = _PyInterpreterState_Get(); - _PyMainInterpreterConfig *config = &interp->config; - - printf("main_config[install_signal_handlers] = %i\n", config->install_signal_handlers); -#define DUMP_ATTR(ATTR) \ - do { \ - if (config->ATTR != NULL) { \ - PySys_FormatStdout("main_config[" #ATTR "] = %R\n", config->ATTR); \ - } \ - else { \ - PySys_FormatStdout("main_config[" #ATTR "] = NULL\n"); \ - } \ - } while (0) - - DUMP_ATTR(argv); - /* FIXME: DUMP_ATTR(executable); */ - DUMP_ATTR(prefix); - DUMP_ATTR(base_prefix); - DUMP_ATTR(base_exec_prefix); - DUMP_ATTR(warnoptions); - DUMP_ATTR(xoptions); - /* FIXME: DUMP_ATTR(module_search_path); */ - DUMP_ATTR(pycache_prefix); - -#undef DUMP_ATTR -} - - static void dump_config(void) { - dump_core_config(); - dump_main_config(); + if (dump_config_impl() < 0) { + fprintf(stderr, "failed to dump the configuration:\n"); + PyErr_Print(); + } } diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 7aa64e11287..3e547c53ab0 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -55,6 +55,78 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif +PyObject * +_Py_GetGlobalVariablesAsDict(void) +{ + PyObject *dict, *obj; + + dict = PyDict_New(); + if (dict == NULL) { + return NULL; + } + +#define SET_ITEM(KEY, EXPR) \ + do { \ + obj = (EXPR); \ + if (obj == NULL) { \ + return NULL; \ + } \ + int res = PyDict_SetItemString(dict, (KEY), obj); \ + Py_DECREF(obj); \ + if (res < 0) { \ + goto fail; \ + } \ + } while (0) +#define SET_ITEM_INT(VAR) \ + SET_ITEM(#VAR, PyLong_FromLong(VAR)) +#define FROM_STRING(STR) \ + ((STR != NULL) ? \ + PyUnicode_FromString(STR) \ + : (Py_INCREF(Py_None), Py_None)) +#define SET_ITEM_STR(VAR) \ + SET_ITEM(#VAR, FROM_STRING(VAR)) + + SET_ITEM_STR(Py_FileSystemDefaultEncoding); + SET_ITEM_INT(Py_HasFileSystemDefaultEncoding); + SET_ITEM_STR(Py_FileSystemDefaultEncodeErrors); + SET_ITEM_INT(_Py_HasFileSystemDefaultEncodeErrors); + + SET_ITEM_INT(Py_UTF8Mode); + SET_ITEM_INT(Py_DebugFlag); + SET_ITEM_INT(Py_VerboseFlag); + SET_ITEM_INT(Py_QuietFlag); + SET_ITEM_INT(Py_InteractiveFlag); + SET_ITEM_INT(Py_InspectFlag); + + SET_ITEM_INT(Py_OptimizeFlag); + SET_ITEM_INT(Py_NoSiteFlag); + SET_ITEM_INT(Py_BytesWarningFlag); + SET_ITEM_INT(Py_FrozenFlag); + SET_ITEM_INT(Py_IgnoreEnvironmentFlag); + SET_ITEM_INT(Py_DontWriteBytecodeFlag); + SET_ITEM_INT(Py_NoUserSiteDirectory); + SET_ITEM_INT(Py_UnbufferedStdioFlag); + SET_ITEM_INT(Py_HashRandomizationFlag); + SET_ITEM_INT(Py_IsolatedFlag); + +#ifdef MS_WINDOWS + SET_ITEM_INT(Py_LegacyWindowsFSEncodingFlag); + SET_ITEM_INT(Py_LegacyWindowsStdioFlag); +#endif + + return dict; + +fail: + Py_DECREF(dict); + return NULL; + +#undef FROM_STRING +#undef SET_ITEM +#undef SET_ITEM_INT +#undef SET_ITEM_STR +} + + void _Py_wstrlist_clear(int len, wchar_t **list) { @@ -493,6 +565,7 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) COPY_FLAG(legacy_windows_fs_encoding, Py_LegacyWindowsFSEncodingFlag); COPY_FLAG(legacy_windows_stdio, Py_LegacyWindowsStdioFlag); #endif + COPY_FLAG(_frozen, Py_FrozenFlag); COPY_NOT_FLAG(use_environment, Py_IgnoreEnvironmentFlag); COPY_NOT_FLAG(buffered_stdio, Py_UnbufferedStdioFlag); @@ -1438,6 +1511,8 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) _Py_wstrlist_as_pylist(config->argc, config->argv)); SET_ITEM("program", FROM_WSTRING(config->program)); + SET_ITEM("xoptions", + _Py_wstrlist_as_pylist(config->nxoption, config->xoptions)); SET_ITEM("warnoptions", _Py_wstrlist_as_pylist(config->nwarnoption, config->warnoptions)); SET_ITEM("module_search_path_env",