bpo-36945: Add _PyPreConfig.configure_locale (GH-13368)

_PyPreConfig_InitIsolatedConfig() sets configure_locale to 0 to
prevent Python to modify the LC_CTYPE locale. In that case,
coerce_c_locale an coerce_c_locale_warn are set to 0 as well.

Include/cpython/coreconfig.h

@@ -79,6 +79,10 @@ typedef struct {
        set to !Py_IgnoreEnvironmentFlag. */
     int use_environment;
 
+    /* Set the LC_CTYPE locale to the user preferred locale? If equals to 0,
+       set coerce_c_locale and coerce_c_locale_warn to 0. */
+    int configure_locale;
+
     /* Coerce the LC_CTYPE locale if it's equal to "C"? (PEP 538)
 
        Set to 0 by PYTHONCOERCECLOCALE=0. Set to 1 by PYTHONCOERCECLOCALE=1.
@@ -147,6 +151,7 @@ typedef struct {
         ._config_version = _Py_CONFIG_VERSION, \
         .isolated = -1, \
         .use_environment = -1, \
+        .configure_locale = 1, \
         .utf8_mode = -2, \
         .dev_mode = -1, \
         .allocator = PYMEM_ALLOCATOR_NOT_SET}

Lib/test/test_embed.py

@@ -272,10 +272,15 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
     maxDiff = 4096
     UTF8_MODE_ERRORS = ('surrogatepass' if MS_WINDOWS else 'surrogateescape')
 
-    # Mark config which should be get by get_default_config()
+    # Marker to read the default configuration: get_default_config()
     GET_DEFAULT_CONFIG = object()
+
+    # Marker to ignore a configuration parameter
+    IGNORE_CONFIG = object()
+
     DEFAULT_PRE_CONFIG = {
         'allocator': PYMEM_ALLOCATOR_NOT_SET,
+        'configure_locale': 1,
         'coerce_c_locale': 0,
         'coerce_c_locale_warn': 0,
         'utf8_mode': 0,
@@ -405,7 +410,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
                 xoptions[opt] = True
         return xoptions
 
-    def get_expected_config(self, expected, env, add_path=None):
+    def get_expected_config(self, expected_preconfig, expected, env, add_path=None):
         expected = dict(self.DEFAULT_CORE_CONFIG, **expected)
 
         code = textwrap.dedent('''
@@ -471,6 +476,14 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
         if add_path is not None:
             expected['module_search_paths'].append(add_path)
+
+        if not expected_preconfig['configure_locale']:
+            # there is no easy way to get the locale encoding before
+            # setlocale(LC_CTYPE, "") is called: don't test encodings
+            for key in ('filesystem_encoding', 'filesystem_errors',
+                        'stdio_encoding', 'stdio_errors'):
+                expected[key] = self.IGNORE_CONFIG
+
         return expected
 
     def check_pre_config(self, config, expected):
@@ -480,6 +493,10 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
 
     def check_core_config(self, config, expected):
         core_config = dict(config['core_config'])
+        for key, value in list(expected.items()):
+            if value is self.IGNORE_CONFIG:
+                del core_config[key]
+                del expected[key]
         self.assertEqual(core_config, expected)
 
     def check_global_config(self, config):
@@ -517,7 +534,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         env['PYTHONUTF8'] = '0'
 
         expected_preconfig = dict(self.DEFAULT_PRE_CONFIG, **expected_preconfig)
-        expected_config = self.get_expected_config(expected_config, env, add_path)
+        expected_config = self.get_expected_config(expected_preconfig, expected_config, env, add_path)
         for key in self.COPY_PRE_CONFIG:
             if key not in expected_preconfig:
                 expected_preconfig[key] = expected_config[key]
@@ -692,7 +709,9 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         self.check_config("preinit_isolated2", config, preconfig)
 
     def test_init_isolated_config(self):
-        preconfig = {}
+        preconfig = {
+            'configure_locale': 0,
+        }
         config = {
             'isolated': 1,
             'use_environment': 0,
@@ -710,6 +729,13 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         }
         self.check_config("init_python_config", config, preconfig)
 
+    def test_init_dont_configure_locale(self):
+        # _PyPreConfig.configure_locale=0
+        preconfig = {
+            'configure_locale': 0,
+        }
+        self.check_config("init_dont_configure_locale", {}, preconfig)
+
     def test_init_read_set(self):
         preconfig = {}
         core_config = {

Programs/_testembed.c

@@ -788,6 +788,33 @@ static int init_python_config(void)
 }
 
 
+static int init_dont_configure_locale(void)
+{
+    _PyInitError err;
+
+    _PyPreConfig preconfig = _PyPreConfig_INIT;
+    preconfig.configure_locale = 0;
+    preconfig.coerce_c_locale = 1;
+    preconfig.coerce_c_locale_warn = 1;
+
+    err = _Py_PreInitialize(&preconfig);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_ExitInitError(err);
+    }
+
+    _PyCoreConfig config = _PyCoreConfig_INIT;
+    config.program_name = L"./_testembed";
+    err = _Py_InitializeFromConfig(&config);
+    if (_Py_INIT_FAILED(err)) {
+        _Py_ExitInitError(err);
+    }
+
+    dump_config();
+    Py_Finalize();
+    return 0;
+}
+
+
 static int init_dev_mode(void)
 {
     _PyCoreConfig config;
@@ -966,6 +993,7 @@ static struct TestCase TestCases[] = {
     { "init_env", test_init_env },
     { "init_env_dev_mode", test_init_env_dev_mode },
     { "init_env_dev_mode_alloc", test_init_env_dev_mode_alloc },
+    { "init_dont_configure_locale", init_dont_configure_locale },
     { "init_dev_mode", init_dev_mode },
     { "init_isolated_flag", init_isolated_flag },
     { "init_isolated_config", init_isolated_config },

Python/preconfig.c

@@ -286,6 +286,7 @@ _PyPreConfig_InitIsolatedConfig(_PyPreConfig *config)
 {
     _PyPreConfig_Init(config);
 
+    config->configure_locale = 0;
     config->isolated = 1;
     config->use_environment = 0;
 #ifdef MS_WINDOWS
@@ -312,6 +313,7 @@ _PyPreConfig_Copy(_PyPreConfig *config, const _PyPreConfig *config2)
 
     COPY_ATTR(isolated);
     COPY_ATTR(use_environment);
+    COPY_ATTR(configure_locale);
     COPY_ATTR(dev_mode);
     COPY_ATTR(coerce_c_locale);
     COPY_ATTR(coerce_c_locale_warn);
@@ -360,6 +362,7 @@ _PyPreConfig_AsDict(const _PyPreConfig *config)
 
     SET_ITEM_INT(isolated);
     SET_ITEM_INT(use_environment);
+    SET_ITEM_INT(configure_locale);
     SET_ITEM_INT(coerce_c_locale);
     SET_ITEM_INT(coerce_c_locale_warn);
     SET_ITEM_INT(utf8_mode);
@@ -603,6 +606,12 @@ preconfig_init_utf8_mode(_PyPreConfig *config, const _PyPreCmdline *cmdline)
 static void
 preconfig_init_coerce_c_locale(_PyPreConfig *config)
 {
+    if (!config->configure_locale) {
+        config->coerce_c_locale = 0;
+        config->coerce_c_locale_warn = 0;
+        return;
+    }
+
     const char *env = _Py_GetEnv(config->use_environment, "PYTHONCOERCECLOCALE");
     if (env) {
         if (strcmp(env, "0") == 0) {
@@ -746,7 +755,9 @@ _PyPreConfig_Read(_PyPreConfig *config, const _PyArgv *args)
     }
 
     /* Set LC_CTYPE to the user preferred locale */
+    if (config->configure_locale) {
         _Py_SetLocaleFromEnv(LC_CTYPE);
+    }
 
     _PyPreCmdline cmdline = _PyPreCmdline_INIT;
     int init_utf8_mode = Py_UTF8Mode;
@@ -879,12 +890,14 @@ _PyPreConfig_Write(const _PyPreConfig *config)
 
     _PyPreConfig_SetGlobalConfig(config);
 
+    if (config->configure_locale) {
         if (config->coerce_c_locale) {
             _Py_CoerceLegacyLocale(config->coerce_c_locale_warn);
         }
 
         /* Set LC_CTYPE to the user preferred locale */
         _Py_SetLocaleFromEnv(LC_CTYPE);
+    }
 
     /* Write the new pre-configuration into _PyRuntime */
     PyMemAllocatorEx old_alloc;