bpo-42260: PyConfig_Read() only parses argv once (GH-23168)

The PyConfig_Read() function now only parses PyConfig.argv arguments
once: PyConfig.parse_argv is set to 2 after arguments are parsed.
Since Python arguments are strippped from PyConfig.argv, parsing
arguments twice would parse the application options as Python
options.

* Rework the PyConfig documentation.
* Fix _testinternalcapi.set_config() error handling.
* SetConfigTests no longer needs parse_argv=0 when restoring the old
  configuration.
This commit is contained in:
Victor Stinner 2020-11-05 18:58:07 +01:00 committed by GitHub
parent f3cb814315
commit dc42af8fd1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 131 additions and 102 deletions

View File

@ -8,55 +8,68 @@ Python Initialization Configuration
.. versionadded:: 3.8 .. versionadded:: 3.8
Structures: Python can be initialized with :c:func:`Py_InitializeFromConfig` and the
:c:type:`PyConfig` structure. It can be preinitialized with
:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure.
* :c:type:`PyConfig` There are two kinds of configuration:
* :c:type:`PyPreConfig`
* :c:type:`PyStatus`
* :c:type:`PyWideStringList`
Functions: * The :ref:`Python Configuration <init-python-config>` can be used to build a
customized Python which behaves as the regular Python. For example,
environments variables and command line arguments are used to configure
Python.
* :c:func:`PyConfig_Clear` * The :ref:`Isolated Configuration <init-isolated-conf>` can be used to embed
* :c:func:`PyConfig_InitIsolatedConfig` Python into an application. It isolates Python from the system. For example,
* :c:func:`PyConfig_InitPythonConfig` environments variables are ignored, the LC_CTYPE locale is left unchanged and
* :c:func:`PyConfig_Read` no signal handler is registred.
* :c:func:`PyConfig_SetArgv`
* :c:func:`PyConfig_SetBytesArgv`
* :c:func:`PyConfig_SetBytesString`
* :c:func:`PyConfig_SetString`
* :c:func:`PyConfig_SetWideStringList`
* :c:func:`PyPreConfig_InitIsolatedConfig`
* :c:func:`PyPreConfig_InitPythonConfig`
* :c:func:`PyStatus_Error`
* :c:func:`PyStatus_Exception`
* :c:func:`PyStatus_Exit`
* :c:func:`PyStatus_IsError`
* :c:func:`PyStatus_IsExit`
* :c:func:`PyStatus_NoMemory`
* :c:func:`PyStatus_Ok`
* :c:func:`PyWideStringList_Append`
* :c:func:`PyWideStringList_Insert`
* :c:func:`Py_ExitStatusException`
* :c:func:`Py_InitializeFromConfig`
* :c:func:`Py_PreInitialize`
* :c:func:`Py_PreInitializeFromArgs`
* :c:func:`Py_PreInitializeFromBytesArgs`
* :c:func:`Py_RunMain`
* :c:func:`Py_GetArgcArgv`
The preconfiguration (``PyPreConfig`` type) is stored in
``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in
``PyInterpreterState.config``.
See also :ref:`Initialization, Finalization, and Threads <initialization>`. See also :ref:`Initialization, Finalization, and Threads <initialization>`.
.. seealso:: .. seealso::
:pep:`587` "Python Initialization Configuration". :pep:`587` "Python Initialization Configuration".
Example
=======
Example of customized Python always running in isolated mode::
int main(int argc, char **argv)
{
PyStatus status;
PyConfig config;
PyConfig_InitPythonConfig(&config);
config.isolated = 1;
/* Decode command line arguments.
Implicitly preinitialize Python (in isolated mode). */
status = PyConfig_SetBytesArgv(&config, argc, argv);
if (PyStatus_Exception(status)) {
goto exception;
}
status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) {
goto exception;
}
PyConfig_Clear(&config);
return Py_RunMain();
exception:
PyConfig_Clear(&config);
if (PyStatus_IsExit(status)) {
return status.exitcode;
}
/* Display the error message and exit the process with
non-zero exit code */
Py_ExitStatusException(status);
}
PyWideStringList PyWideStringList
---------------- ================
.. c:type:: PyWideStringList .. c:type:: PyWideStringList
@ -95,7 +108,7 @@ PyWideStringList
List items. List items.
PyStatus PyStatus
-------- ========
.. c:type:: PyStatus .. c:type:: PyStatus
@ -187,7 +200,7 @@ Example::
PyPreConfig PyPreConfig
----------- ===========
.. c:type:: PyPreConfig .. c:type:: PyPreConfig
@ -317,7 +330,7 @@ PyPreConfig
.. _c-preinit: .. _c-preinit:
Preinitialize Python with PyPreConfig Preinitialize Python with PyPreConfig
------------------------------------- =====================================
The preinitialization of Python: The preinitialization of Python:
@ -326,12 +339,17 @@ The preinitialization of Python:
* Set the :ref:`Python UTF-8 Mode <utf8-mode>` * Set the :ref:`Python UTF-8 Mode <utf8-mode>`
(:c:member:`PyPreConfig.utf8_mode`) (:c:member:`PyPreConfig.utf8_mode`)
The current preconfiguration (``PyPreConfig`` type) is stored in
``_PyRuntime.preconfig``.
Functions to preinitialize Python: Functions to preinitialize Python:
.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) .. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig)
Preinitialize Python from *preconfig* preconfiguration. Preinitialize Python from *preconfig* preconfiguration.
*preconfig* must not be ``NULL``.
.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) .. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv)
Preinitialize Python from *preconfig* preconfiguration. Preinitialize Python from *preconfig* preconfiguration.
@ -339,6 +357,8 @@ Functions to preinitialize Python:
Parse *argv* command line arguments (bytes strings) if Parse *argv* command line arguments (bytes strings) if
:c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero.
*preconfig* must not be ``NULL``.
.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) .. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv)
Preinitialize Python from *preconfig* preconfiguration. Preinitialize Python from *preconfig* preconfiguration.
@ -346,6 +366,8 @@ Functions to preinitialize Python:
Parse *argv* command line arguments (wide strings) if Parse *argv* command line arguments (wide strings) if
:c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero.
*preconfig* must not be ``NULL``.
The caller is responsible to handle exceptions (error or exit) using The caller is responsible to handle exceptions (error or exit) using
:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`.
@ -388,7 +410,7 @@ the :ref:`Python UTF-8 Mode <utf8-mode>`::
PyConfig PyConfig
-------- ========
.. c:type:: PyConfig .. c:type:: PyConfig
@ -449,8 +471,20 @@ PyConfig
Fields which are already initialized are left unchanged. Fields which are already initialized are left unchanged.
The :c:func:`PyConfig_Read` function only parses
:c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv`
is set to ``2`` after arguments are parsed. Since Python arguments are
strippped from :c:member:`PyConfig.argv`, parsing arguments twice would
parse the application options as Python options.
:ref:`Preinitialize Python <c-preinit>` if needed. :ref:`Preinitialize Python <c-preinit>` if needed.
.. versionchanged:: 3.10
The :c:member:`PyConfig.argv` arguments are now only parsed once,
:c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are
parsed, and arguments are only parsed if
:c:member:`PyConfig.parse_argv` equals ``1``.
.. c:function:: void PyConfig_Clear(PyConfig *config) .. c:function:: void PyConfig_Clear(PyConfig *config)
Release configuration memory. Release configuration memory.
@ -833,7 +867,7 @@ PyConfig
If :c:member:`~PyConfig.orig_argv` list is empty and If :c:member:`~PyConfig.orig_argv` list is empty and
:c:member:`~PyConfig.argv` is not a list only containing an empty :c:member:`~PyConfig.argv` is not a list only containing an empty
string, :c:func:`PyConfig_Read()` copies :c:member:`~PyConfig.argv` into string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into
:c:member:`~PyConfig.orig_argv` before modifying :c:member:`~PyConfig.orig_argv` before modifying
:c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is
non-zero). non-zero).
@ -849,12 +883,22 @@ PyConfig
Parse command line arguments? Parse command line arguments?
If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular
Python parses :ref:`command line arguments <using-on-cmdline>`, and strip Python parses :ref:`command line arguments <using-on-cmdline>`, and strip
Python arguments from :c:member:`~PyConfig.argv`. Python arguments from :c:member:`~PyConfig.argv`.
The :c:func:`PyConfig_Read` function only parses
:c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv`
is set to ``2`` after arguments are parsed. Since Python arguments are
strippped from :c:member:`PyConfig.argv`, parsing arguments twice would
parse the application options as Python options.
Default: ``1`` in Python mode, ``0`` in isolated mode. Default: ``1`` in Python mode, ``0`` in isolated mode.
.. versionchanged:: 3.10
The :c:member:`PyConfig.argv` arguments are now only parsed if
:c:member:`PyConfig.parse_argv` equals to ``1``.
.. c:member:: int parser_debug .. c:member:: int parser_debug
Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending
@ -1108,7 +1152,7 @@ the :option:`-X` command line option.
Initialization with PyConfig Initialization with PyConfig
---------------------------- ============================
Function to initialize Python: Function to initialize Python:
@ -1123,6 +1167,9 @@ If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or
:c:func:`PyImport_ExtendInittab` are used, they must be set or called after :c:func:`PyImport_ExtendInittab` are used, they must be set or called after
Python preinitialization and before the Python initialization. Python preinitialization and before the Python initialization.
The current configuration (``PyConfig`` type) is stored in
``PyInterpreterState.config``.
Example setting the program name:: Example setting the program name::
void init_python(void) void init_python(void)
@ -1136,17 +1183,17 @@ Example setting the program name::
status = PyConfig_SetString(&config, &config.program_name, status = PyConfig_SetString(&config, &config.program_name,
L"/path/to/my_program"); L"/path/to/my_program");
if (PyStatus_Exception(status)) { if (PyStatus_Exception(status)) {
goto fail; goto exception;
} }
status = Py_InitializeFromConfig(&config); status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) { if (PyStatus_Exception(status)) {
goto fail; goto exception;
} }
PyConfig_Clear(&config); PyConfig_Clear(&config);
return; return;
fail: exception:
PyConfig_Clear(&config); PyConfig_Clear(&config);
Py_ExitStatusException(status); Py_ExitStatusException(status);
} }
@ -1202,7 +1249,7 @@ configuration, and then override some parameters::
.. _init-isolated-conf: .. _init-isolated-conf:
Isolated Configuration Isolated Configuration
---------------------- ======================
:c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyPreConfig_InitIsolatedConfig` and
:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to
@ -1223,7 +1270,7 @@ configuration.
.. _init-python-config: .. _init-python-config:
Python Configuration Python Configuration
-------------------- ====================
:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig`
functions create a configuration to build a customized Python which behaves as functions create a configuration to build a customized Python which behaves as
@ -1237,46 +1284,11 @@ and :ref:`Python UTF-8 Mode <utf8-mode>`
(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and (:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and
:envvar:`PYTHONCOERCECLOCALE` environment variables. :envvar:`PYTHONCOERCECLOCALE` environment variables.
Example of customized Python always running in isolated mode::
int main(int argc, char **argv)
{
PyStatus status;
PyConfig config;
PyConfig_InitPythonConfig(&config);
config.isolated = 1;
/* Decode command line arguments.
Implicitly preinitialize Python (in isolated mode). */
status = PyConfig_SetBytesArgv(&config, argc, argv);
if (PyStatus_Exception(status)) {
goto fail;
}
status = Py_InitializeFromConfig(&config);
if (PyStatus_Exception(status)) {
goto fail;
}
PyConfig_Clear(&config);
return Py_RunMain();
fail:
PyConfig_Clear(&config);
if (PyStatus_IsExit(status)) {
return status.exitcode;
}
/* Display the error message and exit the process with
non-zero exit code */
Py_ExitStatusException(status);
}
.. _init-path-config: .. _init-path-config:
Path Configuration Path Configuration
------------------ ==================
:c:type:`PyConfig` contains multiple fields for the path configuration: :c:type:`PyConfig` contains multiple fields for the path configuration:
@ -1356,7 +1368,7 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set
Py_RunMain() Py_RunMain()
------------ ============
.. c:function:: int Py_RunMain(void) .. c:function:: int Py_RunMain(void)
@ -1376,7 +1388,7 @@ customized Python always running in isolated mode using
Py_GetArgcArgv() Py_GetArgcArgv()
---------------- ================
.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) .. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv)
@ -1386,7 +1398,7 @@ Py_GetArgcArgv()
Multi-Phase Initialization Private Provisional API Multi-Phase Initialization Private Provisional API
-------------------------------------------------- ==================================================
This section is a private provisional API introducing multi-phase This section is a private provisional API introducing multi-phase
initialization, the core feature of the :pep:`432`: initialization, the core feature of the :pep:`432`:

View File

@ -20,7 +20,7 @@ class SetConfigTests(unittest.TestCase):
self.sys_copy = dict(sys.__dict__) self.sys_copy = dict(sys.__dict__)
def tearDown(self): def tearDown(self):
self.set_config(parse_argv=0) _testinternalcapi.set_config(self.old_config)
sys.__dict__.clear() sys.__dict__.clear()
sys.__dict__.update(self.sys_copy) sys.__dict__.update(self.sys_copy)
@ -234,6 +234,12 @@ class SetConfigTests(unittest.TestCase):
self.assertEqual(sys.argv, ['python_program', 'args']) self.assertEqual(sys.argv, ['python_program', 'args'])
self.assertEqual(sys.orig_argv, ['orig', 'orig_args']) self.assertEqual(sys.orig_argv, ['orig', 'orig_args'])
self.set_config(parse_argv=0,
argv=[],
orig_argv=[])
self.assertEqual(sys.argv, [''])
self.assertEqual(sys.orig_argv, [])
def test_pycache_prefix(self): def test_pycache_prefix(self):
self.check(pycache_prefix=None) self.check(pycache_prefix=None)
self.check(pycache_prefix="pycache_prefix") self.check(pycache_prefix="pycache_prefix")

View File

@ -422,7 +422,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
CONFIG_PYTHON = dict(CONFIG_COMPAT, CONFIG_PYTHON = dict(CONFIG_COMPAT,
_config_init=API_PYTHON, _config_init=API_PYTHON,
configure_c_stdio=1, configure_c_stdio=1,
parse_argv=1, parse_argv=2,
) )
CONFIG_ISOLATED = dict(CONFIG_COMPAT, CONFIG_ISOLATED = dict(CONFIG_COMPAT,
_config_init=API_ISOLATED, _config_init=API_ISOLATED,
@ -800,7 +800,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'-X', 'cmdline_xoption', '-X', 'cmdline_xoption',
'-c', 'pass', '-c', 'pass',
'arg2'], 'arg2'],
'parse_argv': 1, 'parse_argv': 2,
'xoptions': [ 'xoptions': [
'config_xoption1=3', 'config_xoption1=3',
'config_xoption2=', 'config_xoption2=',
@ -1045,7 +1045,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'orig_argv': ['python3', '-c', code, 'arg2'], 'orig_argv': ['python3', '-c', code, 'arg2'],
'program_name': './python3', 'program_name': './python3',
'run_command': code + '\n', 'run_command': code + '\n',
'parse_argv': 1, 'parse_argv': 2,
} }
self.check_all_configs("test_init_run_main", config, api=API_PYTHON) self.check_all_configs("test_init_run_main", config, api=API_PYTHON)
@ -1059,7 +1059,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'arg2'], 'arg2'],
'program_name': './python3', 'program_name': './python3',
'run_command': code + '\n', 'run_command': code + '\n',
'parse_argv': 1, 'parse_argv': 2,
'_init_main': 0, '_init_main': 0,
} }
self.check_all_configs("test_init_main", config, self.check_all_configs("test_init_main", config,
@ -1068,7 +1068,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
def test_init_parse_argv(self): def test_init_parse_argv(self):
config = { config = {
'parse_argv': 1, 'parse_argv': 2,
'argv': ['-c', 'arg1', '-v', 'arg3'], 'argv': ['-c', 'arg1', '-v', 'arg3'],
'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'],
'program_name': './argv0', 'program_name': './argv0',

View File

@ -0,0 +1,5 @@
The :c:func:`PyConfig_Read` function now only parses :c:member:`PyConfig.argv`
arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments
are parsed. Since Python arguments are strippped from
:c:member:`PyConfig.argv`, parsing arguments twice would parse the application
options as Python options.

View File

@ -253,14 +253,17 @@ test_set_config(PyObject *Py_UNUSED(self), PyObject *dict)
PyConfig config; PyConfig config;
PyConfig_InitIsolatedConfig(&config); PyConfig_InitIsolatedConfig(&config);
if (_PyConfig_FromDict(&config, dict) < 0) { if (_PyConfig_FromDict(&config, dict) < 0) {
PyConfig_Clear(&config); goto error;
return NULL;
} }
if (_PyInterpreterState_SetConfig(&config) < 0) { if (_PyInterpreterState_SetConfig(&config) < 0) {
return NULL; goto error;
} }
PyConfig_Clear(&config); PyConfig_Clear(&config);
Py_RETURN_NONE; Py_RETURN_NONE;
error:
PyConfig_Clear(&config);
return NULL;
} }

View File

@ -1325,8 +1325,6 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
GET_UINT(_init_main); GET_UINT(_init_main);
GET_UINT(_isolated_interpreter); GET_UINT(_isolated_interpreter);
assert(config_check_consistency(config));
#undef CHECK_VALUE #undef CHECK_VALUE
#undef GET_UINT #undef GET_UINT
#undef GET_WSTR #undef GET_WSTR
@ -2145,6 +2143,11 @@ config_read(PyConfig *config)
config->configure_c_stdio = 1; config->configure_c_stdio = 1;
} }
// Only parse arguments once.
if (config->parse_argv == 1) {
config->parse_argv = 2;
}
return _PyStatus_OK(); return _PyStatus_OK();
} }
@ -2635,7 +2638,7 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline)
{ {
PyStatus status; PyStatus status;
if (config->parse_argv) { if (config->parse_argv == 1) {
if (_PyWideStringList_Copy(&precmdline->argv, &config->argv) < 0) { if (_PyWideStringList_Copy(&precmdline->argv, &config->argv) < 0) {
return _PyStatus_NO_MEMORY(); return _PyStatus_NO_MEMORY();
} }
@ -2713,7 +2716,7 @@ config_read_cmdline(PyConfig *config)
} }
} }
if (config->parse_argv) { if (config->parse_argv == 1) {
Py_ssize_t opt_index; Py_ssize_t opt_index;
status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index); status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index);
if (_PyStatus_EXCEPTION(status)) { if (_PyStatus_EXCEPTION(status)) {