From bcfbbd704646622e919c1306a91fba61d603483d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 17 May 2019 22:44:16 +0200 Subject: [PATCH] bpo-36945: Add _PyPreConfig.configure_locale (GH-13368) _PyPreConfig_InitIsolatedConfig() sets configure_locale to 0 to prevent Python to modify the LC_CTYPE locale. In that case, coerce_c_locale an coerce_c_locale_warn are set to 0 as well. --- Include/cpython/coreconfig.h | 5 +++++ Lib/test/test_embed.py | 34 ++++++++++++++++++++++++++++++---- Programs/_testembed.c | 28 ++++++++++++++++++++++++++++ Python/preconfig.c | 25 +++++++++++++++++++------ 4 files changed, 82 insertions(+), 10 deletions(-) diff --git a/Include/cpython/coreconfig.h b/Include/cpython/coreconfig.h index 7d561ceb3ee..73861a96d77 100644 --- a/Include/cpython/coreconfig.h +++ b/Include/cpython/coreconfig.h @@ -79,6 +79,10 @@ typedef struct { set to !Py_IgnoreEnvironmentFlag. */ int use_environment; + /* Set the LC_CTYPE locale to the user preferred locale? If equals to 0, + set coerce_c_locale and coerce_c_locale_warn to 0. */ + int configure_locale; + /* Coerce the LC_CTYPE locale if it's equal to "C"? (PEP 538) Set to 0 by PYTHONCOERCECLOCALE=0. Set to 1 by PYTHONCOERCECLOCALE=1. @@ -147,6 +151,7 @@ typedef struct { ._config_version = _Py_CONFIG_VERSION, \ .isolated = -1, \ .use_environment = -1, \ + .configure_locale = 1, \ .utf8_mode = -2, \ .dev_mode = -1, \ .allocator = PYMEM_ALLOCATOR_NOT_SET} diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 50badd8e585..c389df85fb6 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -272,10 +272,15 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): maxDiff = 4096 UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape') - # Mark config which should be get by get_default_config() + # Marker to read the default configuration: get_default_config() GET_DEFAULT_CONFIG = object() + + # Marker to ignore a configuration parameter + IGNORE_CONFIG = object() + DEFAULT_PRE_CONFIG = { 'allocator': PYMEM_ALLOCATOR_NOT_SET, + 'configure_locale': 1, 'coerce_c_locale': 0, 'coerce_c_locale_warn': 0, 'utf8_mode': 0, @@ -405,7 +410,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): xoptions[opt] = True return xoptions - def get_expected_config(self, expected, env, add_path=None): + def get_expected_config(self, expected_preconfig, expected, env, add_path=None): expected = dict(self.DEFAULT_CORE_CONFIG, **expected) code = textwrap.dedent(''' @@ -471,6 +476,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): if add_path is not None: expected['module_search_paths'].append(add_path) + + if not expected_preconfig['configure_locale']: + # there is no easy way to get the locale encoding before + # setlocale(LC_CTYPE, "") is called: don't test encodings + for key in ('filesystem_encoding', 'filesystem_errors', + 'stdio_encoding', 'stdio_errors'): + expected[key] = self.IGNORE_CONFIG + return expected def check_pre_config(self, config, expected): @@ -480,6 +493,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): def check_core_config(self, config, expected): core_config = dict(config['core_config']) + for key, value in list(expected.items()): + if value is self.IGNORE_CONFIG: + del core_config[key] + del expected[key] self.assertEqual(core_config, expected) def check_global_config(self, config): @@ -517,7 +534,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): env['PYTHONUTF8'] = '0' expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig) - expected_config = self.get_expected_config(expected_config, env, add_path) + expected_config = self.get_expected_config(expected_preconfig, expected_config, env, add_path) for key in self.COPY_PRE_CONFIG: if key not in expected_preconfig: expected_preconfig[key] = expected_config[key] @@ -692,7 +709,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_config("preinit_isolated2", config, preconfig) def test_init_isolated_config(self): - preconfig = {} + preconfig = { + 'configure_locale': 0, + } config = { 'isolated': 1, 'use_environment': 0, @@ -710,6 +729,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): } self.check_config("init_python_config", config, preconfig) + def test_init_dont_configure_locale(self): + # _PyPreConfig.configure_locale=0 + preconfig = { + 'configure_locale': 0, + } + self.check_config("init_dont_configure_locale", {}, preconfig) + def test_init_read_set(self): preconfig = {} core_config = { diff --git a/Programs/_testembed.c b/Programs/_testembed.c index e6896966f53..000b2fcec98 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -788,6 +788,33 @@ static int init_python_config(void) } +static int init_dont_configure_locale(void) +{ + _PyInitError err; + + _PyPreConfig preconfig = _PyPreConfig_INIT; + preconfig.configure_locale = 0; + preconfig.coerce_c_locale = 1; + preconfig.coerce_c_locale_warn = 1; + + err = _Py_PreInitialize(&preconfig); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + _PyCoreConfig config = _PyCoreConfig_INIT; + config.program_name = L"./_testembed"; + err = _Py_InitializeFromConfig(&config); + if (_Py_INIT_FAILED(err)) { + _Py_ExitInitError(err); + } + + dump_config(); + Py_Finalize(); + return 0; +} + + static int init_dev_mode(void) { _PyCoreConfig config; @@ -966,6 +993,7 @@ static struct TestCase TestCases[] = { { "init_env", test_init_env }, { "init_env_dev_mode", test_init_env_dev_mode }, { "init_env_dev_mode_alloc", test_init_env_dev_mode_alloc }, + { "init_dont_configure_locale", init_dont_configure_locale }, { "init_dev_mode", init_dev_mode }, { "init_isolated_flag", init_isolated_flag }, { "init_isolated_config", init_isolated_config }, diff --git a/Python/preconfig.c b/Python/preconfig.c index 7814ee08a63..985af39cb00 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -286,6 +286,7 @@ _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config) { _PyPreConfig_Init(config); + config->configure_locale = 0; config->isolated = 1; config->use_environment = 0; #ifdef MS_WINDOWS @@ -312,6 +313,7 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2) COPY_ATTR(isolated); COPY_ATTR(use_environment); + COPY_ATTR(configure_locale); COPY_ATTR(dev_mode); COPY_ATTR(coerce_c_locale); COPY_ATTR(coerce_c_locale_warn); @@ -360,6 +362,7 @@ _PyPreConfig_AsDict(const _PyPreConfig *config) SET_ITEM_INT(isolated); SET_ITEM_INT(use_environment); + SET_ITEM_INT(configure_locale); SET_ITEM_INT(coerce_c_locale); SET_ITEM_INT(coerce_c_locale_warn); SET_ITEM_INT(utf8_mode); @@ -603,6 +606,12 @@ preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline) static void preconfig_init_coerce_c_locale(_PyPreConfig *config) { + if (!config->configure_locale) { + config->coerce_c_locale = 0; + config->coerce_c_locale_warn = 0; + return; + } + const char *env = _Py_GetEnv(config->use_environment, "PYTHONCOERCECLOCALE"); if (env) { if (strcmp(env, "0") == 0) { @@ -746,7 +755,9 @@ _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args) } /* Set LC_CTYPE to the user preferred locale */ - _Py_SetLocaleFromEnv(LC_CTYPE); + if (config->configure_locale) { + _Py_SetLocaleFromEnv(LC_CTYPE); + } _PyPreCmdline cmdline = _PyPreCmdline_INIT; int init_utf8_mode = Py_UTF8Mode; @@ -879,12 +890,14 @@ _PyPreConfig_Write(const _PyPreConfig *config) _PyPreConfig_SetGlobalConfig(config); - if (config->coerce_c_locale) { - _Py_CoerceLegacyLocale(config->coerce_c_locale_warn); - } + if (config->configure_locale) { + if (config->coerce_c_locale) { + _Py_CoerceLegacyLocale(config->coerce_c_locale_warn); + } - /* Set LC_CTYPE to the user preferred locale */ - _Py_SetLocaleFromEnv(LC_CTYPE); + /* Set LC_CTYPE to the user preferred locale */ + _Py_SetLocaleFromEnv(LC_CTYPE); + } /* Write the new pre-configuration into _PyRuntime */ PyMemAllocatorEx old_alloc;