From 65ef7425a32ee411d8047a4fad0fc6bb9ff733b1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 28 Aug 2018 13:51:20 +0200 Subject: [PATCH] bpo-34527: POSIX locale enables the UTF-8 Mode (GH-8972) (GH-8974) * The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C" locale. * On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE locale is "C". * test_utf8_mode.test_cmd_line() checks also that the command line arguments are decoded from UTF-8 when the the UTF-8 Mode is enabled with POSIX locale or C locale. (cherry picked from commit 5cb258950ce9b69b1f65646431c464c0c17b1510) --- Lib/test/test_utf8_mode.py | 23 ++++++++++++++----- .../2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst | 2 ++ .../2018-08-28-11-53-39.bpo-34527.aBEX9b.rst | 3 +++ Modules/main.c | 22 +++++++++++++----- Python/fileutils.c | 2 +- 5 files changed, 39 insertions(+), 13 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst create mode 100644 Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index 26e2e13ec53..4a16b730468 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -12,6 +12,7 @@ from test.support.script_helper import assert_python_ok, assert_python_failure MS_WINDOWS = (sys.platform == 'win32') +POSIX_LOCALES = ('C', 'POSIX') class UTF8ModeTests(unittest.TestCase): @@ -23,7 +24,7 @@ class UTF8ModeTests(unittest.TestCase): def posix_locale(self): loc = locale.setlocale(locale.LC_CTYPE, None) - return (loc == 'C') + return (loc in POSIX_LOCALES) def get_output(self, *args, failure=False, **kw): kw = dict(self.DEFAULT_ENV, **kw) @@ -39,8 +40,10 @@ class UTF8ModeTests(unittest.TestCase): def test_posix_locale(self): code = 'import sys; print(sys.flags.utf8_mode)' - out = self.get_output('-c', code, LC_ALL='C') - self.assertEqual(out, '1') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + out = self.get_output('-c', code, LC_ALL=loc) + self.assertEqual(out, '1') def test_xoption(self): code = 'import sys; print(sys.flags.utf8_mode)' @@ -201,8 +204,10 @@ class UTF8ModeTests(unittest.TestCase): out = self.get_output('-X', 'utf8', '-c', code) self.assertEqual(out, 'UTF-8 UTF-8') - out = self.get_output('-X', 'utf8', '-c', code, LC_ALL='C') - self.assertEqual(out, 'UTF-8 UTF-8') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + out = self.get_output('-X', 'utf8', '-c', code, LC_ALL=loc) + self.assertEqual(out, 'UTF-8 UTF-8') @unittest.skipIf(MS_WINDOWS, 'test specific to Unix') def test_cmd_line(self): @@ -217,11 +222,17 @@ class UTF8ModeTests(unittest.TestCase): self.assertEqual(args, ascii(expected), out) check('utf8', [arg_utf8]) + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + check('utf8', [arg_utf8], LC_ALL=loc) + if sys.platform == 'darwin' or support.is_android: c_arg = arg_utf8 else: c_arg = arg_ascii - check('utf8=0', [c_arg], LC_ALL='C') + for loc in POSIX_LOCALES: + with self.subTest(LC_ALL=loc): + check('utf8=0', [c_arg], LC_ALL=loc) def test_optim_level(self): # CPython: check that Py_Main() doesn't increment Py_OptimizeFlag diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst new file mode 100644 index 00000000000..280a8922edc --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-52-13.bpo-34527.sh5MQJ.rst @@ -0,0 +1,2 @@ +The UTF-8 Mode is now also enabled by the "POSIX" locale, not only by the "C" +locale. diff --git a/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst new file mode 100644 index 00000000000..9fce794305c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2018-08-28-11-53-39.bpo-34527.aBEX9b.rst @@ -0,0 +1,3 @@ +On FreeBSD, Py_DecodeLocale() and Py_EncodeLocale() now also forces the +ASCII encoding if the LC_CTYPE locale is "POSIX", not only if the LC_CTYPE +locale is "C". diff --git a/Modules/main.c b/Modules/main.c index 45148dc9839..856f92c9d9b 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -2102,15 +2102,25 @@ done: static void config_init_locale(_PyCoreConfig *config) { - if (_Py_LegacyLocaleDetected()) { - /* POSIX locale: enable C locale coercion and UTF-8 Mode */ - if (config->utf8_mode < 0) { - config->utf8_mode = 1; - } - if (config->coerce_c_locale < 0) { + if (config->coerce_c_locale < 0) { + /* The C locale enables the C locale coercion (PEP 538) */ + if (_Py_LegacyLocaleDetected()) { config->coerce_c_locale = 1; } } + +#ifndef MS_WINDOWS + if (config->utf8_mode < 0) { + /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ + const char *ctype_loc = setlocale(LC_CTYPE, NULL); + if (ctype_loc != NULL + && (strcmp(ctype_loc, "C") == 0 + || strcmp(ctype_loc, "POSIX") == 0)) + { + config->utf8_mode = 1; + } + } +#endif } diff --git a/Python/fileutils.c b/Python/fileutils.c index 35869c81ac9..b413f4e1e68 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -128,7 +128,7 @@ check_force_ascii(void) loc = setlocale(LC_CTYPE, NULL); if (loc == NULL) goto error; - if (strcmp(loc, "C") != 0) { + if (strcmp(loc, "C") != 0 && strcmp(loc, "POSIX") != 0) { /* the LC_CTYPE locale is different than C */ return 0; }