Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-36763: Fix encoding/locale tests in test_embed (GH-13443) 

* Fix encoding and locale tests in test_embed.InitConfigTests.
* InitConfigTests now only computes EXPECTED_CONFIG once.
* Add tests for PYTHONWARNINGS and PYTHONPATH env vars
This commit is contained in:
Victor Stinner 2019-05-20 16:38:48 +02:00 committed by GitHub
parent 9932fd91e8
commit 425717fee1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 112 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

188
Lib/test/test_embed.py
View File

@ -296,6 +296,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
}) })
PYTHON_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG, PYTHON_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG,
parse_argv=1, parse_argv=1,
coerce_c_locale=GET_DEFAULT_CONFIG,
utf8_mode=GET_DEFAULT_CONFIG,
) )
ISOLATED_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG, ISOLATED_PRE_CONFIG = dict(DEFAULT_PRE_CONFIG,
configure_locale=0, configure_locale=0,
@ -303,6 +305,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
use_environment=0, use_environment=0,
utf8_mode=0, utf8_mode=0,
dev_mode=0, dev_mode=0,
coerce_c_locale=0,
) )
COPY_PRE_CONFIG = [ COPY_PRE_CONFIG = [
@ -435,6 +438,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'), ('Py_LegacyWindowsStdioFlag', 'legacy_windows_stdio'),
)) ))
EXPECTED_CONFIG = None
def main_xoptions(self, xoptions_list): def main_xoptions(self, xoptions_list):
xoptions = {} xoptions = {}
for opt in xoptions_list: for opt in xoptions_list:
@ -445,37 +450,15 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
xoptions[opt] = True xoptions[opt] = True
return xoptions return xoptions
def get_expected_config(self, expected_preconfig, expected, env, api, def _get_expected_config(self, env):
add_path=None):
if api == CONFIG_INIT_PYTHON:
default_config = self.PYTHON_CORE_CONFIG
elif api == CONFIG_INIT_ISOLATED:
default_config = self.ISOLATED_CORE_CONFIG
else:
default_config = self.DEFAULT_CORE_CONFIG
expected = dict(default_config, **expected)
expected['_config_init'] = api
code = textwrap.dedent(''' code = textwrap.dedent('''
import json import json
import sys import sys
import _testinternalcapi import _testinternalcapi
configs = _testinternalcapi.get_configs() configs = _testinternalcapi.get_configs()
core_config = configs['core_config']
data = {
'stdio_encoding': sys.stdout.encoding,
'stdio_errors': sys.stdout.errors,
'prefix': sys.prefix,
'base_prefix': sys.base_prefix,
'exec_prefix': sys.exec_prefix,
'base_exec_prefix': sys.base_exec_prefix,
'filesystem_encoding': sys.getfilesystemencoding(),
'filesystem_errors': sys.getfilesystemencodeerrors(),
'module_search_paths': core_config['module_search_paths'],
}
data = json.dumps(data) data = json.dumps(configs)
data = data.encode('utf-8') data = data.encode('utf-8')
sys.stdout.buffer.write(data) sys.stdout.buffer.write(data)
sys.stdout.buffer.flush() sys.stdout.buffer.flush()
@ -484,10 +467,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
# Use -S to not import the site module: get the proper configuration # Use -S to not import the site module: get the proper configuration
# when test_embed is run from a venv (bpo-35313) # when test_embed is run from a venv (bpo-35313)
args = [sys.executable, '-S', '-c', code] args = [sys.executable, '-S', '-c', code]
env = dict(env)
if not expected['isolated']:
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'
proc = subprocess.run(args, env=env, proc = subprocess.run(args, env=env,
stdout=subprocess.PIPE, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) stderr=subprocess.STDOUT)
@ -496,10 +475,46 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
f"stdout={proc.stdout!r} stderr={proc.stderr!r}") f"stdout={proc.stdout!r} stderr={proc.stderr!r}")
stdout = proc.stdout.decode('utf-8') stdout = proc.stdout.decode('utf-8')
try: try:
config = json.loads(stdout) return json.loads(stdout)
except json.JSONDecodeError: except json.JSONDecodeError:
self.fail(f"fail to decode stdout: {stdout!r}") self.fail(f"fail to decode stdout: {stdout!r}")
def get_expected_config(self, expected_preconfig, expected, env, api,
add_path=None):
cls = self.__class__
if cls.EXPECTED_CONFIG is None:
cls.EXPECTED_CONFIG = self._get_expected_config(env)
configs = {key: dict(value)
for key, value in self.EXPECTED_CONFIG.items()}
pre_config = configs['pre_config']
for key, value in expected_preconfig.items():
if value is self.GET_DEFAULT_CONFIG:
expected_preconfig[key] = pre_config[key]
if not expected_preconfig['configure_locale'] or api == CONFIG_INIT:
# there is no easy way to get the locale encoding before
# setlocale(LC_CTYPE, "") is called: don't test encodings
for key in ('filesystem_encoding', 'filesystem_errors',
'stdio_encoding', 'stdio_errors'):
expected[key] = self.IGNORE_CONFIG
if not expected_preconfig['configure_locale']:
# UTF-8 Mode depends on the locale. There is no easy way
# to guess if UTF-8 Mode will be enabled or not if the locale
# is not configured.
expected_preconfig['utf8_mode'] = self.IGNORE_CONFIG
if expected_preconfig['utf8_mode'] == 1:
if expected['filesystem_encoding'] is self.GET_DEFAULT_CONFIG:
expected['filesystem_encoding'] = 'utf-8'
if expected['filesystem_errors'] is self.GET_DEFAULT_CONFIG:
expected['filesystem_errors'] = self.UTF8_MODE_ERRORS
if expected['stdio_encoding'] is self.GET_DEFAULT_CONFIG:
expected['stdio_encoding'] = 'utf-8'
if expected['stdio_errors'] is self.GET_DEFAULT_CONFIG:
expected['stdio_errors'] = 'surrogateescape'
if expected['executable'] is self.GET_DEFAULT_CONFIG: if expected['executable'] is self.GET_DEFAULT_CONFIG:
if sys.platform == 'win32': if sys.platform == 'win32':
expected['executable'] = self.test_exe expected['executable'] = self.test_exe
@ -511,24 +526,28 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
if expected['program_name'] is self.GET_DEFAULT_CONFIG: if expected['program_name'] is self.GET_DEFAULT_CONFIG:
expected['program_name'] = './_testembed' expected['program_name'] = './_testembed'
core_config = configs['core_config']
for key, value in expected.items(): for key, value in expected.items():
if value is self.GET_DEFAULT_CONFIG: if value is self.GET_DEFAULT_CONFIG:
expected[key] = config[key] expected[key] = core_config[key]
prepend_path = expected['module_search_path_env']
if prepend_path is not None:
expected['module_search_paths'] = [prepend_path, *expected['module_search_paths']]
if add_path is not None: if add_path is not None:
expected['module_search_paths'].append(add_path) expected['module_search_paths'] = [*expected['module_search_paths'], add_path]
if not expected_preconfig['configure_locale']: for key in self.COPY_PRE_CONFIG:
# there is no easy way to get the locale encoding before if key not in expected_preconfig:
# setlocale(LC_CTYPE, "") is called: don't test encodings expected_preconfig[key] = expected[key]
for key in ('filesystem_encoding', 'filesystem_errors',
'stdio_encoding', 'stdio_errors'):
expected[key] = self.IGNORE_CONFIG
return expected
def check_pre_config(self, config, expected): def check_pre_config(self, config, expected):
self.assertEqual(config['pre_config'], expected) pre_config = dict(config['pre_config'])
for key, value in list(expected.items()):
if value is self.IGNORE_CONFIG:
del pre_config[key]
del expected[key]
self.assertEqual(pre_config, expected)
def check_core_config(self, config, expected): def check_core_config(self, config, expected):
core_config = dict(config['core_config']) core_config = dict(config['core_config'])
@ -567,10 +586,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
for key in list(env): for key in list(env):
if key.startswith('PYTHON'): if key.startswith('PYTHON'):
del env[key] del env[key]
# Disable C locale coercion and UTF-8 mode to not depend
# on the current locale
env['PYTHONCOERCECLOCALE'] = '0'
env['PYTHONUTF8'] = '0'
if api == CONFIG_INIT_ISOLATED: if api == CONFIG_INIT_ISOLATED:
default_preconfig = self.ISOLATED_PRE_CONFIG default_preconfig = self.ISOLATED_PRE_CONFIG
@ -583,12 +598,19 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
expected_preconfig = dict(default_preconfig, **expected_preconfig) expected_preconfig = dict(default_preconfig, **expected_preconfig)
if expected_config is None: if expected_config is None:
expected_config = {} expected_config = {}
expected_config = self.get_expected_config(expected_preconfig,
expected_config, env, if api == CONFIG_INIT_PYTHON:
api, add_path) default_config = self.PYTHON_CORE_CONFIG
for key in self.COPY_PRE_CONFIG: elif api == CONFIG_INIT_ISOLATED:
if key not in expected_preconfig: default_config = self.ISOLATED_CORE_CONFIG
expected_preconfig[key] = expected_config[key] else:
default_config = self.DEFAULT_CORE_CONFIG
expected_config = dict(default_config, **expected_config)
expected_config['_config_init'] = api
self.get_expected_config(expected_preconfig,
expected_config, env,
api, add_path)
out, err = self.run_embedded_interpreter(testname, env=env) out, err = self.run_embedded_interpreter(testname, env=env)
if stderr is None and not expected_config['verbose']: if stderr is None and not expected_config['verbose']:
@ -624,10 +646,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'quiet': 1, 'quiet': 1,
'buffered_stdio': 0, 'buffered_stdio': 0,
'stdio_encoding': 'utf-8',
'stdio_errors': 'surrogateescape',
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,
'user_site_directory': 0, 'user_site_directory': 0,
'pathconfig_warnings': 0, 'pathconfig_warnings': 0,
} }
@ -650,8 +668,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'stdio_encoding': 'iso8859-1', 'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace', 'stdio_errors': 'replace',
'filesystem_encoding': 'utf-8',
'filesystem_errors': self.UTF8_MODE_ERRORS,
'pycache_prefix': 'conf_pycache_prefix', 'pycache_prefix': 'conf_pycache_prefix',
'program_name': './conf_program_name', 'program_name': './conf_program_name',
@ -679,43 +695,42 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
} }
self.check_config("init_from_config", config, preconfig) self.check_config("init_from_config", config, preconfig)
INIT_ENV_PRECONFIG = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
}
INIT_ENV_CONFIG = {
'use_hash_seed': 1,
'hash_seed': 42,
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'inspect': 1,
'optimization_level': 2,
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
}
def test_init_env(self): def test_init_env(self):
self.check_config("init_env", self.INIT_ENV_CONFIG, self.INIT_ENV_PRECONFIG) preconfig = {
'allocator': PYMEM_ALLOCATOR_MALLOC,
}
config = {
'use_hash_seed': 1,
'hash_seed': 42,
'tracemalloc': 2,
'import_time': 1,
'malloc_stats': 1,
'inspect': 1,
'optimization_level': 2,
'module_search_path_env': '/my/path',
'pycache_prefix': 'env_pycache_prefix',
'write_bytecode': 0,
'verbose': 1,
'buffered_stdio': 0,
'stdio_encoding': 'iso8859-1',
'stdio_errors': 'replace',
'user_site_directory': 0,
'faulthandler': 1,
'warnoptions': ['EnvVar'],
}
self.check_config("init_env", config, preconfig)
def test_init_env_dev_mode(self): def test_init_env_dev_mode(self):
preconfig = dict(self.INIT_ENV_PRECONFIG, preconfig = dict(allocator=PYMEM_ALLOCATOR_DEBUG)
allocator=PYMEM_ALLOCATOR_DEBUG) config = dict(dev_mode=1,
config = dict(self.INIT_ENV_CONFIG, faulthandler=1,
dev_mode=1,
warnoptions=['default']) warnoptions=['default'])
self.check_config("init_env_dev_mode", config, preconfig) self.check_config("init_env_dev_mode", config, preconfig)
def test_init_env_dev_mode_alloc(self): def test_init_env_dev_mode_alloc(self):
preconfig = dict(self.INIT_ENV_PRECONFIG, preconfig = dict(allocator=PYMEM_ALLOCATOR_MALLOC)
allocator=PYMEM_ALLOCATOR_MALLOC) config = dict(dev_mode=1,
config = dict(self.INIT_ENV_CONFIG, faulthandler=1,
dev_mode=1,
warnoptions=['default']) warnoptions=['default'])
self.check_config("init_env_dev_mode_alloc", config, preconfig) self.check_config("init_env_dev_mode_alloc", config, preconfig)
@ -800,6 +815,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
# _PyPreConfig.configure_locale=0 # _PyPreConfig.configure_locale=0
preconfig = { preconfig = {
'configure_locale': 0, 'configure_locale': 0,
'coerce_c_locale': 0,
} }
self.check_config("init_dont_configure_locale", {}, preconfig, self.check_config("init_dont_configure_locale", {}, preconfig,
api=CONFIG_INIT_PYTHON) api=CONFIG_INIT_PYTHON)

19
Programs/_testembed.c
View File

@ -568,7 +568,7 @@ static int test_init_dont_parse_argv(void)
} }
static void set_all_env_vars(void) static void set_most_env_vars(void)
{ {
putenv("PYTHONHASHSEED=42"); putenv("PYTHONHASHSEED=42");
putenv("PYTHONMALLOC=malloc"); putenv("PYTHONMALLOC=malloc");
@ -585,13 +585,15 @@ static void set_all_env_vars(void)
putenv("PYTHONNOUSERSITE=1"); putenv("PYTHONNOUSERSITE=1");
putenv("PYTHONFAULTHANDLER=1"); putenv("PYTHONFAULTHANDLER=1");
putenv("PYTHONIOENCODING=iso8859-1:replace"); putenv("PYTHONIOENCODING=iso8859-1:replace");
/* FIXME: test PYTHONWARNINGS */ }
/* FIXME: test PYTHONEXECUTABLE */
/* FIXME: test PYTHONHOME */
/* FIXME: test PYTHONDEBUG */ static void set_all_env_vars(void)
/* FIXME: test PYTHONDUMPREFS */ {
/* FIXME: test PYTHONCOERCECLOCALE */ set_most_env_vars();
/* FIXME: test PYTHONPATH */
putenv("PYTHONWARNINGS=EnvVar");
putenv("PYTHONPATH=/my/path");
} }
@ -609,7 +611,6 @@ static int test_init_env(void)
static void set_all_env_vars_dev_mode(void) static void set_all_env_vars_dev_mode(void)
{ {
set_all_env_vars();
putenv("PYTHONMALLOC="); putenv("PYTHONMALLOC=");
putenv("PYTHONFAULTHANDLER="); putenv("PYTHONFAULTHANDLER=");
putenv("PYTHONDEVMODE=1"); putenv("PYTHONDEVMODE=1");