From dc42af8fd16b10127ce1fc93c13bc1bfd2674aa2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2020 18:58:07 +0100 Subject: [PATCH] bpo-42260: PyConfig_Read() only parses argv once (GH-23168) The PyConfig_Read() function now only parses PyConfig.argv arguments once: PyConfig.parse_argv is set to 2 after arguments are parsed. Since Python arguments are strippped from PyConfig.argv, parsing arguments twice would parse the application options as Python options. * Rework the PyConfig documentation. * Fix _testinternalcapi.set_config() error handling. * SetConfigTests no longer needs parse_argv=0 when restoring the old configuration. --- Doc/c-api/init_config.rst | 190 ++++++++++-------- Lib/test/_test_embed_set_config.py | 8 +- Lib/test/test_embed.py | 10 +- .../2020-11-05-18-02-07.bpo-42260.pAeaNR.rst | 5 + Modules/_testinternalcapi.c | 9 +- Python/initconfig.c | 11 +- 6 files changed, 131 insertions(+), 102 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c957d6c0f72..edfeba5db7d 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -8,55 +8,68 @@ Python Initialization Configuration .. versionadded:: 3.8 -Structures: +Python can be initialized with :c:func:`Py_InitializeFromConfig` and the +:c:type:`PyConfig` structure. It can be preinitialized with +:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure. -* :c:type:`PyConfig` -* :c:type:`PyPreConfig` -* :c:type:`PyStatus` -* :c:type:`PyWideStringList` +There are two kinds of configuration: -Functions: +* The :ref:`Python Configuration ` can be used to build a + customized Python which behaves as the regular Python. For example, + environments variables and command line arguments are used to configure + Python. -* :c:func:`PyConfig_Clear` -* :c:func:`PyConfig_InitIsolatedConfig` -* :c:func:`PyConfig_InitPythonConfig` -* :c:func:`PyConfig_Read` -* :c:func:`PyConfig_SetArgv` -* :c:func:`PyConfig_SetBytesArgv` -* :c:func:`PyConfig_SetBytesString` -* :c:func:`PyConfig_SetString` -* :c:func:`PyConfig_SetWideStringList` -* :c:func:`PyPreConfig_InitIsolatedConfig` -* :c:func:`PyPreConfig_InitPythonConfig` -* :c:func:`PyStatus_Error` -* :c:func:`PyStatus_Exception` -* :c:func:`PyStatus_Exit` -* :c:func:`PyStatus_IsError` -* :c:func:`PyStatus_IsExit` -* :c:func:`PyStatus_NoMemory` -* :c:func:`PyStatus_Ok` -* :c:func:`PyWideStringList_Append` -* :c:func:`PyWideStringList_Insert` -* :c:func:`Py_ExitStatusException` -* :c:func:`Py_InitializeFromConfig` -* :c:func:`Py_PreInitialize` -* :c:func:`Py_PreInitializeFromArgs` -* :c:func:`Py_PreInitializeFromBytesArgs` -* :c:func:`Py_RunMain` -* :c:func:`Py_GetArgcArgv` - -The preconfiguration (``PyPreConfig`` type) is stored in -``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in -``PyInterpreterState.config``. +* The :ref:`Isolated Configuration ` can be used to embed + Python into an application. It isolates Python from the system. For example, + environments variables are ignored, the LC_CTYPE locale is left unchanged and + no signal handler is registred. See also :ref:`Initialization, Finalization, and Threads `. .. seealso:: :pep:`587` "Python Initialization Configuration". +Example +======= + +Example of customized Python always running in isolated mode:: + + int main(int argc, char **argv) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto exception; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); + + return Py_RunMain(); + + exception: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } + PyWideStringList ----------------- +================ .. c:type:: PyWideStringList @@ -95,7 +108,7 @@ PyWideStringList List items. PyStatus --------- +======== .. c:type:: PyStatus @@ -187,7 +200,7 @@ Example:: PyPreConfig ------------ +=========== .. c:type:: PyPreConfig @@ -317,7 +330,7 @@ PyPreConfig .. _c-preinit: Preinitialize Python with PyPreConfig -------------------------------------- +===================================== The preinitialization of Python: @@ -326,12 +339,17 @@ The preinitialization of Python: * Set the :ref:`Python UTF-8 Mode ` (:c:member:`PyPreConfig.utf8_mode`) +The current preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig``. + Functions to preinitialize Python: .. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) Preinitialize Python from *preconfig* preconfiguration. + *preconfig* must not be ``NULL``. + .. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) Preinitialize Python from *preconfig* preconfiguration. @@ -339,6 +357,8 @@ Functions to preinitialize Python: Parse *argv* command line arguments (bytes strings) if :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + *preconfig* must not be ``NULL``. + .. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) Preinitialize Python from *preconfig* preconfiguration. @@ -346,6 +366,8 @@ Functions to preinitialize Python: Parse *argv* command line arguments (wide strings) if :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + *preconfig* must not be ``NULL``. + The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -388,7 +410,7 @@ the :ref:`Python UTF-8 Mode `:: PyConfig --------- +======== .. c:type:: PyConfig @@ -449,8 +471,20 @@ PyConfig Fields which are already initialized are left unchanged. + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. + :ref:`Preinitialize Python ` if needed. + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed once, + :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are + parsed, and arguments are only parsed if + :c:member:`PyConfig.parse_argv` equals ``1``. + .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. @@ -833,7 +867,7 @@ PyConfig If :c:member:`~PyConfig.orig_argv` list is empty and :c:member:`~PyConfig.argv` is not a list only containing an empty - string, :c:func:`PyConfig_Read()` copies :c:member:`~PyConfig.argv` into + string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into :c:member:`~PyConfig.orig_argv` before modifying :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is non-zero). @@ -849,12 +883,22 @@ PyConfig Parse command line arguments? - If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular + If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular Python parses :ref:`command line arguments `, and strip Python arguments from :c:member:`~PyConfig.argv`. + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. + Default: ``1`` in Python mode, ``0`` in isolated mode. + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed if + :c:member:`PyConfig.parse_argv` equals to ``1``. + .. c:member:: int parser_debug Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending @@ -1108,7 +1152,7 @@ the :option:`-X` command line option. Initialization with PyConfig ----------------------------- +============================ Function to initialize Python: @@ -1123,6 +1167,9 @@ If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or :c:func:`PyImport_ExtendInittab` are used, they must be set or called after Python preinitialization and before the Python initialization. +The current configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. + Example setting the program name:: void init_python(void) @@ -1136,17 +1183,17 @@ Example setting the program name:: status = PyConfig_SetString(&config, &config.program_name, L"/path/to/my_program"); if (PyStatus_Exception(status)) { - goto fail; + goto exception; } status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { - goto fail; + goto exception; } PyConfig_Clear(&config); return; - fail: + exception: PyConfig_Clear(&config); Py_ExitStatusException(status); } @@ -1202,7 +1249,7 @@ configuration, and then override some parameters:: .. _init-isolated-conf: Isolated Configuration ----------------------- +====================== :c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to @@ -1223,7 +1270,7 @@ configuration. .. _init-python-config: Python Configuration --------------------- +==================== :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` functions create a configuration to build a customized Python which behaves as @@ -1237,46 +1284,11 @@ and :ref:`Python UTF-8 Mode ` (:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and :envvar:`PYTHONCOERCECLOCALE` environment variables. -Example of customized Python always running in isolated mode:: - - int main(int argc, char **argv) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config.isolated = 1; - - /* Decode command line arguments. - Implicitly preinitialize Python (in isolated mode). */ - status = PyConfig_SetBytesArgv(&config, argc, argv); - if (PyStatus_Exception(status)) { - goto fail; - } - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto fail; - } - PyConfig_Clear(&config); - - return Py_RunMain(); - - fail: - PyConfig_Clear(&config); - if (PyStatus_IsExit(status)) { - return status.exitcode; - } - /* Display the error message and exit the process with - non-zero exit code */ - Py_ExitStatusException(status); - } - .. _init-path-config: Path Configuration ------------------- +================== :c:type:`PyConfig` contains multiple fields for the path configuration: @@ -1356,7 +1368,7 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set Py_RunMain() ------------- +============ .. c:function:: int Py_RunMain(void) @@ -1376,7 +1388,7 @@ customized Python always running in isolated mode using Py_GetArgcArgv() ----------------- +================ .. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) @@ -1386,7 +1398,7 @@ Py_GetArgcArgv() Multi-Phase Initialization Private Provisional API --------------------------------------------------- +================================================== This section is a private provisional API introducing multi-phase initialization, the core feature of the :pep:`432`: diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index 7c913811ded..a19f8db1584 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -20,7 +20,7 @@ class SetConfigTests(unittest.TestCase): self.sys_copy = dict(sys.__dict__) def tearDown(self): - self.set_config(parse_argv=0) + _testinternalcapi.set_config(self.old_config) sys.__dict__.clear() sys.__dict__.update(self.sys_copy) @@ -234,6 +234,12 @@ class SetConfigTests(unittest.TestCase): self.assertEqual(sys.argv, ['python_program', 'args']) self.assertEqual(sys.orig_argv, ['orig', 'orig_args']) + self.set_config(parse_argv=0, + argv=[], + orig_argv=[]) + self.assertEqual(sys.argv, ['']) + self.assertEqual(sys.orig_argv, []) + def test_pycache_prefix(self): self.check(pycache_prefix=None) self.check(pycache_prefix="pycache_prefix") diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 91820615193..a7d912178a2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -422,7 +422,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, configure_c_stdio=1, - parse_argv=1, + parse_argv=2, ) CONFIG_ISOLATED = dict(CONFIG_COMPAT, _config_init=API_ISOLATED, @@ -800,7 +800,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '-X', 'cmdline_xoption', '-c', 'pass', 'arg2'], - 'parse_argv': 1, + 'parse_argv': 2, 'xoptions': [ 'config_xoption1=3', 'config_xoption2=', @@ -1045,7 +1045,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'orig_argv': ['python3', '-c', code, 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 1, + 'parse_argv': 2, } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1059,7 +1059,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 1, + 'parse_argv': 2, '_init_main': 0, } self.check_all_configs("test_init_main", config, @@ -1068,7 +1068,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): def test_init_parse_argv(self): config = { - 'parse_argv': 1, + 'parse_argv': 2, 'argv': ['-c', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', diff --git a/Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst b/Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst new file mode 100644 index 00000000000..0d6a277db88 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst @@ -0,0 +1,5 @@ +The :c:func:`PyConfig_Read` function now only parses :c:member:`PyConfig.argv` +arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments +are parsed. Since Python arguments are strippped from +:c:member:`PyConfig.argv`, parsing arguments twice would parse the application +options as Python options. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index be144bfba02..df4725ea0a1 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -253,14 +253,17 @@ test_set_config(PyObject *Py_UNUSED(self), PyObject *dict) PyConfig config; PyConfig_InitIsolatedConfig(&config); if (_PyConfig_FromDict(&config, dict) < 0) { - PyConfig_Clear(&config); - return NULL; + goto error; } if (_PyInterpreterState_SetConfig(&config) < 0) { - return NULL; + goto error; } PyConfig_Clear(&config); Py_RETURN_NONE; + +error: + PyConfig_Clear(&config); + return NULL; } diff --git a/Python/initconfig.c b/Python/initconfig.c index d54d5b7a999..e0811b56cb3 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1325,8 +1325,6 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_init_main); GET_UINT(_isolated_interpreter); - assert(config_check_consistency(config)); - #undef CHECK_VALUE #undef GET_UINT #undef GET_WSTR @@ -2145,6 +2143,11 @@ config_read(PyConfig *config) config->configure_c_stdio = 1; } + // Only parse arguments once. + if (config->parse_argv == 1) { + config->parse_argv = 2; + } + return _PyStatus_OK(); } @@ -2635,7 +2638,7 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline) { PyStatus status; - if (config->parse_argv) { + if (config->parse_argv == 1) { if (_PyWideStringList_Copy(&precmdline->argv, &config->argv) < 0) { return _PyStatus_NO_MEMORY(); } @@ -2713,7 +2716,7 @@ config_read_cmdline(PyConfig *config) } } - if (config->parse_argv) { + if (config->parse_argv == 1) { Py_ssize_t opt_index; status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index); if (_PyStatus_EXCEPTION(status)) {