From 7a0791b6992d420dc52536257f2f093851ed7215 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 17 Sep 2018 16:22:29 -0700 Subject: [PATCH] bpo-34589: C locale coercion off by default (GH-9073) Py_Initialize() and Py_Main() cannot enable the C locale coercion (PEP 538) anymore: it is always disabled. It can now only be enabled by the Python program ("python3). test_embed: get_filesystem_encoding() doesn't have to set PYTHONUTF8 nor PYTHONCOERCECLOCALE, these variables are already set in the parent. --- Include/coreconfig.h | 11 ++++++- Include/pylifecycle.h | 4 +++ Lib/test/test_embed.py | 4 --- .../2018-09-18-00-09-31.bpo-34589.C7bUpq.rst | 3 ++ Modules/main.c | 29 +++++++++++++++---- Programs/_testembed.c | 5 +--- Programs/python.c | 2 +- Python/coreconfig.c | 7 ++++- 8 files changed, 49 insertions(+), 16 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2018-09-18-00-09-31.bpo-34589.C7bUpq.rst diff --git a/Include/coreconfig.h b/Include/coreconfig.h index 8944ec23df0..293d8ed1660 100644 --- a/Include/coreconfig.h +++ b/Include/coreconfig.h @@ -301,6 +301,10 @@ typedef struct { variable. The option is also enabled if the LC_CTYPE locale is "C" and a target locale (ex: "C.UTF-8") is supported by the platform. + Py_Initialize() and Py_Main() must not enable C locale coercion: it is + always disabled. The option can only be enabled by the Python program + ("python3). + See also the _coerce_c_locale_warn option. */ int _coerce_c_locale; @@ -308,6 +312,10 @@ typedef struct { Enabled by the PYTHONCOERCECLOCALE=warn environment variable. + Py_Initialize() and Py_Main() must not enable C locale coercion warning: + it is always disabled. The warning can only be enabled by the Python + program ("python3). + See also the _coerce_c_locale option. */ int _coerce_c_locale_warn; @@ -328,7 +336,8 @@ typedef struct { .use_hash_seed = -1, \ .faulthandler = -1, \ .tracemalloc = -1, \ - ._coerce_c_locale = -1, \ + ._coerce_c_locale = 0, \ + ._coerce_c_locale_warn = 0, \ .utf8_mode = -1, \ .argc = -1, \ .nmodule_search_path = -1, \ diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h index 04e672e96e1..f64bae3f617 100644 --- a/Include/pylifecycle.h +++ b/Include/pylifecycle.h @@ -83,7 +83,11 @@ PyAPI_FUNC(int) Py_FdIsInteractive(FILE *, const char *); /* Bootstrap __main__ (defined in Modules/main.c) */ PyAPI_FUNC(int) Py_Main(int argc, wchar_t **argv); #ifdef Py_BUILD_CORE +# ifdef MS_WINDOWS +PyAPI_FUNC(int) _Py_WindowsMain(int argc, wchar_t **argv); +# else PyAPI_FUNC(int) _Py_UnixMain(int argc, char **argv); +# endif #endif /* In getpath.c */ diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index c00000e6813..e531fd49d5a 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -324,10 +324,6 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'print(sys.getfilesystemencoding(), ' 'sys.getfilesystemencodeerrors())') args = (sys.executable, '-c', code) - env = dict(env) - if not isolated: - env['PYTHONCOERCECLOCALE'] = '0' - env['PYTHONUTF8'] = '0' proc = subprocess.run(args, text=True, env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) diff --git a/Misc/NEWS.d/next/C API/2018-09-18-00-09-31.bpo-34589.C7bUpq.rst b/Misc/NEWS.d/next/C API/2018-09-18-00-09-31.bpo-34589.C7bUpq.rst new file mode 100644 index 00000000000..27b6a6e0017 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2018-09-18-00-09-31.bpo-34589.C7bUpq.rst @@ -0,0 +1,3 @@ +Py_Initialize() and Py_Main() cannot enable the C locale coercion (PEP 538) +anymore: it is always disabled. It can now only be enabled by the Python +program ("python3). diff --git a/Modules/main.c b/Modules/main.c index 6bc2917e2cc..d4ae4a0ae30 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -1700,7 +1700,8 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) static int -pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) +pymain_init(_PyMain *pymain, PyInterpreterState **interp_p, + int use_c_locale_coercion) { /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, @@ -1713,6 +1714,11 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) _PyCoreConfig local_config = _PyCoreConfig_INIT; _PyCoreConfig *config = &local_config; + if (use_c_locale_coercion) { + /* set to -1 to be able to enable the feature */ + config->_coerce_c_locale = -1; + config->_coerce_c_locale_warn = -1; + } _PyCoreConfig_GetGlobalConfig(config); @@ -1747,10 +1753,10 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) static int -pymain_main(_PyMain *pymain) +pymain_main(_PyMain *pymain, int use_c_locale_coercion) { PyInterpreterState *interp; - int res = pymain_init(pymain, &interp); + int res = pymain_init(pymain, &interp, use_c_locale_coercion); if (res != 1) { if (pymain_run_python(pymain, interp) < 0) { _Py_FatalInitError(pymain->err); @@ -1777,10 +1783,22 @@ Py_Main(int argc, wchar_t **argv) pymain.argc = argc; pymain.wchar_argv = argv; - return pymain_main(&pymain); + return pymain_main(&pymain, 0); } +#ifdef MS_WINDOWS +int +_Py_WindowsMain(int argc, wchar_t **argv) +{ + _PyMain pymain = _PyMain_INIT; + pymain.use_bytes_argv = 0; + pymain.argc = argc; + pymain.wchar_argv = argv; + + return pymain_main(&pymain, 1); +} +#else int _Py_UnixMain(int argc, char **argv) { @@ -1789,8 +1807,9 @@ _Py_UnixMain(int argc, char **argv) pymain.argc = argc; pymain.bytes_argv = argv; - return pymain_main(&pymain); + return pymain_main(&pymain, 1); } +#endif /* this is gonna seem *real weird*, but if you put some other code between diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 12880325b25..b9394690c3f 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -482,8 +482,6 @@ static int test_init_from_config(void) putenv("PYTHONMALLOCSTATS=0"); config.malloc_stats = 1; - /* FIXME: test _coerce_c_locale and _coerce_c_locale_warn */ - putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; config.utf8_mode = 1; @@ -606,8 +604,7 @@ static int test_init_isolated(void) /* Test _PyCoreConfig.isolated=1 */ _PyCoreConfig config = _PyCoreConfig_INIT; - /* Set _coerce_c_locale and utf8_mode to not depend on the locale */ - config._coerce_c_locale = 0; + /* Set utf8_mode to not depend on the locale */ config.utf8_mode = 0; /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; diff --git a/Programs/python.c b/Programs/python.c index 78e48f800c9..c7697facbe3 100644 --- a/Programs/python.c +++ b/Programs/python.c @@ -6,7 +6,7 @@ int wmain(int argc, wchar_t **argv) { - return Py_Main(argc, argv); + return _Py_WindowsMain(argc, argv); } #else int diff --git a/Python/coreconfig.c b/Python/coreconfig.c index 1e03ce31f3d..131a043ff28 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -816,7 +816,9 @@ config_read_env_vars(_PyCoreConfig *config) } } else if (strcmp(env, "warn") == 0) { - config->_coerce_c_locale_warn = 1; + if (config->_coerce_c_locale_warn < 0) { + config->_coerce_c_locale_warn = 1; + } } else { if (config->_coerce_c_locale < 0) { @@ -1324,6 +1326,9 @@ _PyCoreConfig_Read(_PyCoreConfig *config) if (config->_coerce_c_locale < 0) { config->_coerce_c_locale = 0; } + if (config->_coerce_c_locale_warn < 0) { + config->_coerce_c_locale_warn = 0; + } if (config->utf8_mode < 0) { config->utf8_mode = 0; }