From 64366fa9b3ba71b8a503a8719eff433f4ea49eb9 Mon Sep 17 00:00:00 2001 From: Julien Danjou Date: Mon, 2 Nov 2020 15:16:25 +0100 Subject: [PATCH 01/61] bpo-41435: Add sys._current_exceptions() function (GH-21689) This adds a new function named sys._current_exceptions() which is equivalent ot sys._current_frames() except that it returns the exceptions currently handled by other threads. It is equivalent to calling sys.exc_info() for each running thread. --- Doc/library/sys.rst | 12 ++++ Include/cpython/pystate.h | 5 ++ Lib/test/test_sys.py | 67 +++++++++++++++++++ .../2020-08-07-13-42-48.bpo-41435.qPWjJA.rst | 1 + Python/clinic/sysmodule.c.h | 22 +++++- Python/pystate.c | 63 +++++++++++++++++ Python/sysmodule.c | 16 +++++ 7 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 2f0840e2a74..f0acfcfe639 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -196,6 +196,18 @@ always available. .. audit-event:: sys._current_frames "" sys._current_frames +.. function:: _current_exceptions() + + Return a dictionary mapping each thread's identifier to the topmost exception + currently active in that thread at the time the function is called. + If a thread is not currently handling an exception, it is not included in + the result dictionary. + + This is most useful for statistical profiling. + + This function should be used for internal and specialized purposes only. + + .. audit-event:: sys._current_exceptions "" sys._current_exceptions .. function:: breakpointhook() diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 5d5e4e33197..25522b4dbec 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -167,6 +167,11 @@ PyAPI_FUNC(PyInterpreterState *) _PyGILState_GetInterpreterStateUnsafe(void); */ PyAPI_FUNC(PyObject *) _PyThread_CurrentFrames(void); +/* The implementation of sys._current_exceptions() Returns a dict mapping + thread id to that thread's current exception. +*/ +PyAPI_FUNC(PyObject *) _PyThread_CurrentExceptions(void); + /* Routines for advanced debuggers, requested by David Beazley. Don't use unless you know what you are doing! */ PyAPI_FUNC(PyInterpreterState *) PyInterpreterState_Main(void); diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 30c29a26a99..332ed8f550c 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -432,6 +432,73 @@ class SysModuleTest(unittest.TestCase): leave_g.set() t.join() + @threading_helper.reap_threads + def test_current_exceptions(self): + import threading + import traceback + + # Spawn a thread that blocks at a known place. Then the main + # thread does sys._current_frames(), and verifies that the frames + # returned make sense. + entered_g = threading.Event() + leave_g = threading.Event() + thread_info = [] # the thread's id + + def f123(): + g456() + + def g456(): + thread_info.append(threading.get_ident()) + entered_g.set() + while True: + try: + raise ValueError("oops") + except ValueError: + if leave_g.wait(timeout=support.LONG_TIMEOUT): + break + + t = threading.Thread(target=f123) + t.start() + entered_g.wait() + + # At this point, t has finished its entered_g.set(), although it's + # impossible to guess whether it's still on that line or has moved on + # to its leave_g.wait(). + self.assertEqual(len(thread_info), 1) + thread_id = thread_info[0] + + d = sys._current_exceptions() + for tid in d: + self.assertIsInstance(tid, int) + self.assertGreater(tid, 0) + + main_id = threading.get_ident() + self.assertIn(main_id, d) + self.assertIn(thread_id, d) + self.assertEqual((None, None, None), d.pop(main_id)) + + # Verify that the captured thread frame is blocked in g456, called + # from f123. This is a litte tricky, since various bits of + # threading.py are also in the thread's call stack. + exc_type, exc_value, exc_tb = d.pop(thread_id) + stack = traceback.extract_stack(exc_tb.tb_frame) + for i, (filename, lineno, funcname, sourceline) in enumerate(stack): + if funcname == "f123": + break + else: + self.fail("didn't find f123() on thread's call stack") + + self.assertEqual(sourceline, "g456()") + + # And the next record must be for g456(). + filename, lineno, funcname, sourceline = stack[i+1] + self.assertEqual(funcname, "g456") + self.assertTrue(sourceline.startswith("if leave_g.wait(")) + + # Reap the spawned thread. + leave_g.set() + t.join() + def test_attributes(self): self.assertIsInstance(sys.api_version, int) self.assertIsInstance(sys.argv, list) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst b/Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst new file mode 100644 index 00000000000..d2978f9b4ec --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst @@ -0,0 +1 @@ +Add `sys._current_exceptions()` function to retrieve a dictionary mapping each thread's identifier to the topmost exception currently active in that thread at the time the function is called. \ No newline at end of file diff --git a/Python/clinic/sysmodule.c.h b/Python/clinic/sysmodule.c.h index c1a9a2d69f0..addd58922e7 100644 --- a/Python/clinic/sysmodule.c.h +++ b/Python/clinic/sysmodule.c.h @@ -801,6 +801,26 @@ sys__current_frames(PyObject *module, PyObject *Py_UNUSED(ignored)) return sys__current_frames_impl(module); } +PyDoc_STRVAR(sys__current_exceptions__doc__, +"_current_exceptions($module, /)\n" +"--\n" +"\n" +"Return a dict mapping each thread\'s identifier to its current raised exception.\n" +"\n" +"This function should be used for specialized purposes only."); + +#define SYS__CURRENT_EXCEPTIONS_METHODDEF \ + {"_current_exceptions", (PyCFunction)sys__current_exceptions, METH_NOARGS, sys__current_exceptions__doc__}, + +static PyObject * +sys__current_exceptions_impl(PyObject *module); + +static PyObject * +sys__current_exceptions(PyObject *module, PyObject *Py_UNUSED(ignored)) +{ + return sys__current_exceptions_impl(module); +} + PyDoc_STRVAR(sys_call_tracing__doc__, "call_tracing($module, func, args, /)\n" "--\n" @@ -945,4 +965,4 @@ sys_getandroidapilevel(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF #define SYS_GETANDROIDAPILEVEL_METHODDEF #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */ -/*[clinic end generated code: output=87baa3357293ea65 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bbc4963fe86a29d9 input=a9049054013a1b77]*/ diff --git a/Python/pystate.c b/Python/pystate.c index e88898670cd..e37cbd5a657 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -1222,6 +1222,69 @@ done: return result; } +PyObject * +_PyThread_CurrentExceptions(void) +{ + PyThreadState *tstate = _PyThreadState_GET(); + + _Py_EnsureTstateNotNULL(tstate); + + if (_PySys_Audit(tstate, "sys._current_exceptions", NULL) < 0) { + return NULL; + } + + PyObject *result = PyDict_New(); + if (result == NULL) { + return NULL; + } + + /* for i in all interpreters: + * for t in all of i's thread states: + * if t's frame isn't NULL, map t's id to its frame + * Because these lists can mutate even when the GIL is held, we + * need to grab head_mutex for the duration. + */ + _PyRuntimeState *runtime = tstate->interp->runtime; + HEAD_LOCK(runtime); + PyInterpreterState *i; + for (i = runtime->interpreters.head; i != NULL; i = i->next) { + PyThreadState *t; + for (t = i->tstate_head; t != NULL; t = t->next) { + _PyErr_StackItem *err_info = _PyErr_GetTopmostException(t); + if (err_info == NULL) { + continue; + } + PyObject *id = PyLong_FromUnsignedLong(t->thread_id); + if (id == NULL) { + goto fail; + } + PyObject *exc_info = PyTuple_Pack( + 3, + err_info->exc_type != NULL ? err_info->exc_type : Py_None, + err_info->exc_value != NULL ? err_info->exc_value : Py_None, + err_info->exc_traceback != NULL ? err_info->exc_traceback : Py_None); + if (exc_info == NULL) { + Py_DECREF(id); + goto fail; + } + int stat = PyDict_SetItem(result, id, exc_info); + Py_DECREF(id); + Py_DECREF(exc_info); + if (stat < 0) { + goto fail; + } + } + } + goto done; + +fail: + Py_CLEAR(result); + +done: + HEAD_UNLOCK(runtime); + return result; +} + /* Python "auto thread state" API. */ /* Keep this as a static, as it is not reliable! It can only diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 749b96455d6..945e639ca57 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1837,6 +1837,21 @@ sys__current_frames_impl(PyObject *module) return _PyThread_CurrentFrames(); } +/*[clinic input] +sys._current_exceptions + +Return a dict mapping each thread's identifier to its current raised exception. + +This function should be used for specialized purposes only. +[clinic start generated code]*/ + +static PyObject * +sys__current_exceptions_impl(PyObject *module) +/*[clinic end generated code: output=2ccfd838c746f0ba input=0e91818fbf2edc1f]*/ +{ + return _PyThread_CurrentExceptions(); +} + /*[clinic input] sys.call_tracing @@ -1953,6 +1968,7 @@ static PyMethodDef sys_methods[] = { METH_FASTCALL | METH_KEYWORDS, breakpointhook_doc}, SYS__CLEAR_TYPE_CACHE_METHODDEF SYS__CURRENT_FRAMES_METHODDEF + SYS__CURRENT_EXCEPTIONS_METHODDEF SYS_DISPLAYHOOK_METHODDEF SYS_EXC_INFO_METHODDEF SYS_EXCEPTHOOK_METHODDEF From 301822859b3fc34801a06f1090d62f9f2ee5b092 Mon Sep 17 00:00:00 2001 From: Lysandros Nikolaou Date: Mon, 2 Nov 2020 17:27:30 +0200 Subject: [PATCH 02/61] bpo-42224: Fix test_format when locale does not expect number grouping (GH-23067) --- Lib/test/test_format.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_format.py b/Lib/test/test_format.py index d2744cdfdca..9653e46ecc5 100644 --- a/Lib/test/test_format.py +++ b/Lib/test/test_format.py @@ -428,13 +428,16 @@ class FormatTest(unittest.TestCase): localeconv = locale.localeconv() sep = localeconv['thousands_sep'] point = localeconv['decimal_point'] + grouping = localeconv['grouping'] text = format(123456789, "n") - self.assertIn(sep, text) + if grouping: + self.assertIn(sep, text) self.assertEqual(text.replace(sep, ''), '123456789') text = format(1234.5, "n") - self.assertIn(sep, text) + if grouping: + self.assertIn(sep, text) self.assertIn(point, text) self.assertEqual(text.replace(sep, ''), '1234' + point + '5') finally: From 4b9aad49992a825d8c76e428ed1aca81dd3878b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Nov 2020 16:49:54 +0100 Subject: [PATCH 03/61] bpo-42236: Enhance init and encoding documentation (GH-23109) Enhance the documentation of the Python startup, filesystem encoding and error handling, locale encoding. Add a new "Python UTF-8 Mode" section. * Add "locale encoding" and "filesystem encoding and error handler" to the glossary * Remove documentation from Include/cpython/initconfig.h: move it to Doc/c-api/init_config.rst. * Doc/c-api/init_config.rst: * Document command line options and environment variables * Document default values. * Add a new "Python UTF-8 Mode" section in Doc/library/os.rst. * Add warnings to Py_DecodeLocale() and Py_EncodeLocale() docs. * Document how Python selects the filesystem encoding and error handler at a single place: PyConfig.filesystem_encoding and PyConfig.filesystem_errors. * PyConfig: move orig_argv member at the right place. --- Doc/c-api/exceptions.rst | 8 +- Doc/c-api/init.rst | 5 +- Doc/c-api/init_config.rst | 589 +++++++++++++++++++++++++++-------- Doc/c-api/sys.rst | 66 ++-- Doc/c-api/unicode.rst | 11 +- Doc/c-api/veryhigh.rst | 21 +- Doc/glossary.rst | 31 ++ Doc/howto/unicode.rst | 10 +- Doc/library/devmode.rst | 3 + Doc/library/exceptions.rst | 4 +- Doc/library/locale.rst | 26 +- Doc/library/os.rst | 90 +++++- Doc/library/sys.rst | 42 ++- Doc/using/cmdline.rst | 58 +--- Doc/using/windows.rst | 24 +- Doc/whatsnew/3.7.rst | 3 +- Include/cpython/initconfig.h | 253 ++------------- Python/initconfig.c | 1 + Python/preconfig.c | 10 +- 19 files changed, 735 insertions(+), 520 deletions(-) diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst index 247b6d68ece..4e99a0167a6 100644 --- a/Doc/c-api/exceptions.rst +++ b/Doc/c-api/exceptions.rst @@ -182,8 +182,8 @@ For convenience, some of these functions will always return a .. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the filesystem encoding - (:func:`os.fsdecode`). + is given as a C string. *filename* is decoded from the :term:`filesystem + encoding and error handler`. .. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) @@ -266,7 +266,7 @@ For convenience, some of these functions will always return a .. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). + decoded from the :term:`filesystem encoding and error handler`. .. versionadded:: 3.2 @@ -343,7 +343,7 @@ an error value). Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and *module* are UTF-8 encoded strings, and *filename* is decoded from the - filesystem encoding (:func:`os.fsdecode`). + :term:`filesystem encoding and error handler`. .. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 7f06648bcb4..3ce689203a8 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -151,8 +151,9 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. .. c:var:: int Py_LegacyWindowsFSEncodingFlag - If the flag is non-zero, use the ``mbcs`` encoding instead of the UTF-8 - encoding for the filesystem encoding. + If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error + handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, + for the :term:`filesystem encoding and error handler`. Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable is set to a non-empty string. diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index 92a6c3a56d6..dad1f90bea5 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -189,11 +189,7 @@ PyPreConfig .. c:type:: PyPreConfig - Structure used to preinitialize Python: - - * Set the Python memory allocator - * Configure the LC_CTYPE locale - * Set the UTF-8 mode + Structure used to preinitialize Python. Function to initialize a preconfiguration: @@ -211,7 +207,7 @@ PyPreConfig .. c:member:: int allocator - Name of the memory allocator: + Name of the Python memory allocators: * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators (use defaults) @@ -231,27 +227,48 @@ PyPreConfig See :ref:`Memory Management `. + Default: ``PYMEM_ALLOCATOR_NOT_SET``. + .. c:member:: int configure_locale - Set the LC_CTYPE locale to the user preferred locale? If equals to 0, set - :c:member:`coerce_c_locale` and :c:member:`coerce_c_locale_warn` to 0. + Set the LC_CTYPE locale to the user preferred locale? + + If equals to 0, set :c:member:`~PyPreConfig.coerce_c_locale` and + :c:member:`~PyPreConfig.coerce_c_locale_warn` members to 0. + + See the :term:`locale encoding`. + + Default: ``1`` in Python config, ``0`` in isolated config. .. c:member:: int coerce_c_locale - If equals to 2, coerce the C locale; if equals to 1, read the LC_CTYPE - locale to decide if it should be coerced. + If equals to 2, coerce the C locale. + + If equals to 1, read the LC_CTYPE locale to decide if it should be + coerced. + + See the :term:`locale encoding`. + + Default: ``-1`` in Python config, ``0`` in isolated config. .. c:member:: int coerce_c_locale_warn If non-zero, emit a warning if the C locale is coerced. + Default: ``-1`` in Python config, ``0`` in isolated config. + .. c:member:: int dev_mode - See :c:member:`PyConfig.dev_mode`. + If non-zero, enables the :ref:`Python Development Mode `: + see :c:member:`PyConfig.dev_mode`. + + Default: ``-1`` in Python mode, ``0`` in isolated mode. .. c:member:: int isolated - See :c:member:`PyConfig.isolated`. + Isolated mode: see :c:member:`PyConfig.isolated`. + + Default: ``0`` in Python mode, ``1`` in isolated mode. .. c:member:: int legacy_windows_fs_encoding @@ -267,6 +284,8 @@ PyPreConfig Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for Windows specific code. + Default: ``0``. + .. c:member:: int parse_argv If non-zero, :c:func:`Py_PreInitializeFromArgs` and @@ -274,16 +293,36 @@ PyPreConfig same way the regular Python parses command line arguments: see :ref:`Command Line Arguments `. + Default: ``1`` in Python config, ``0`` in isolated config. + .. c:member:: int use_environment - See :c:member:`PyConfig.use_environment`. + Use :ref:`environment variables `? See + :c:member:`PyConfig.use_environment`. + + Default: ``1`` in Python config and ``0`` in isolated config. .. c:member:: int utf8_mode - If non-zero, enable the UTF-8 mode. + If non-zero, enable the :ref:`Python UTF-8 Mode `. -Preinitialization with PyPreConfig ----------------------------------- + Set by the :option:`-X utf8 <-X>` command line option and the + :envvar:`PYTHONUTF8` environment variable. + + Default: ``-1`` in Python config and ``0`` in isolated config. + + +.. _c-preinit: + +Preinitialize Python with PyPreConfig +------------------------------------- + +The preinitialization of Python: + +* Set the Python memory allocators (:c:member:`PyPreConfig.allocator`) +* Configure the LC_CTYPE locale (:term:`locale encoding`) +* Set the :ref:`Python UTF-8 Mode ` + (:c:member:`PyPreConfig.utf8_mode`) Functions to preinitialize Python: @@ -293,13 +332,17 @@ Functions to preinitialize Python: .. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) - Preinitialize Python from *preconfig* preconfiguration and command line - arguments (bytes strings). + Preinitialize Python from *preconfig* preconfiguration. + + Parse *argv* command line arguments (bytes strings) if + :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. .. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) - Preinitialize Python from *preconfig* preconfiguration and command line - arguments (wide strings). + Preinitialize Python from *preconfig* preconfiguration. + + Parse *argv* command line arguments (wide strings) if + :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -309,7 +352,7 @@ For :ref:`Python Configuration ` command line arguments, the command line arguments must also be passed to preinitialize Python, since they have an effect on the pre-configuration like encodings. For example, the :option:`-X utf8 <-X>` command line option -enables the UTF-8 Mode. +enables the :ref:`Python UTF-8 Mode `. ``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. @@ -317,11 +360,12 @@ It can be called before :c:func:`Py_PreInitialize` if :c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be -used before Python preinitialization, whereas calling directly ``malloc()`` and -``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called before -the preinitialization. +used before the Python preinitialization, whereas calling directly ``malloc()`` +and ``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called +before the Python preinitialization. -Example using the preinitialization to enable the UTF-8 Mode:: +Example using the preinitialization to enable +the :ref:`Python UTF-8 Mode `:: PyStatus status; PyPreConfig preconfig; @@ -334,7 +378,7 @@ Example using the preinitialization to enable the UTF-8 Mode:: Py_ExitStatusException(status); } - /* at this point, Python will speak UTF-8 */ + /* at this point, Python speaks UTF-8 */ Py_Initialize(); /* ... use Python API here ... */ @@ -348,47 +392,54 @@ PyConfig Structure containing most parameters to configure Python. + When done, the :c:func:`PyConfig_Clear` function must be used to release the + configuration memory. + Structure methods: .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) - Initialize configuration with :ref:`Python Configuration + Initialize configuration with the :ref:`Python Configuration `. .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) - Initialize configuration with :ref:`Isolated Configuration + Initialize configuration with the :ref:`Isolated Configuration `. .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) Copy the wide character string *str* into ``*config_str``. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) - Decode *str* using ``Py_DecodeLocale()`` and set the result into ``*config_str``. + Decode *str* using :c:func:`Py_DecodeLocale` and set the result into + ``*config_str``. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) - Set command line arguments from wide character strings. + Set command line arguments (:c:member:`~PyConfig.argv` member of + *config*) from the *argv* list of wide character strings. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) - Set command line arguments: decode bytes using :c:func:`Py_DecodeLocale`. + Set command line arguments (:c:member:`~PyConfig.argv` member of + *config*) from the *argv* list of bytes strings. Decode bytes using + :c:func:`Py_DecodeLocale`. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) Set the list of wide strings *list* to *length* and *items*. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: PyStatus PyConfig_Read(PyConfig *config) @@ -396,24 +447,25 @@ PyConfig Fields which are already initialized are left unchanged. - Preinitialize Python if needed. + :ref:`Preinitialize Python ` if needed. .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. - Most ``PyConfig`` methods preinitialize Python if needed. In that case, the - Python preinitialization configuration in based on the :c:type:`PyConfig`. - If configuration fields which are in common with :c:type:`PyPreConfig` are - tuned, they must be set before calling a :c:type:`PyConfig` method: + Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. + In that case, the Python preinitialization configuration + (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration + fields which are in common with :c:type:`PyPreConfig` are tuned, they must + be set before calling a :c:type:`PyConfig` method: - * :c:member:`~PyConfig.dev_mode` - * :c:member:`~PyConfig.isolated` - * :c:member:`~PyConfig.parse_argv` - * :c:member:`~PyConfig.use_environment` + * :c:member:`PyConfig.dev_mode` + * :c:member:`PyConfig.isolated` + * :c:member:`PyConfig.parse_argv` + * :c:member:`PyConfig.use_environment` Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` - is used, this method must be called first, before other methods, since the + is used, this method must be called before other methods, since the preinitialization configuration depends on command line arguments (if :c:member:`parse_argv` is non-zero). @@ -424,11 +476,17 @@ PyConfig .. c:member:: PyWideStringList argv - Command line arguments, :data:`sys.argv`. See - :c:member:`~PyConfig.parse_argv` to parse :c:member:`~PyConfig.argv` the - same way the regular Python parses Python command line arguments. If - :c:member:`~PyConfig.argv` is empty, an empty string is added to ensure - that :data:`sys.argv` always exists and is never empty. + Command line arguments: :data:`sys.argv`. + + Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse + :c:member:`~PyConfig.argv` the same way the regular Python parses Python + command line arguments and then to strip Python arguments from + :c:member:`~PyConfig.argv`. + + If :c:member:`~PyConfig.argv` is empty, an empty string is added to + ensure that :data:`sys.argv` always exists and is never empty. + + Default: ``NULL``. See also the :c:member:`~PyConfig.orig_argv` member. @@ -436,76 +494,136 @@ PyConfig :data:`sys.base_exec_prefix`. + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` output. + .. c:member:: wchar_t* base_executable - :data:`sys._base_executable`: ``__PYVENV_LAUNCHER__`` environment - variable value, or copy of :c:member:`PyConfig.executable`. + Python base executable: :data:`sys._base_executable`. + + Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. + + Set from :c:member:`PyConfig.executable` if ``NULL``. + + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` output. .. c:member:: wchar_t* base_prefix :data:`sys.base_prefix`. - .. c:member:: wchar_t* platlibdir + Default: ``NULL``. - :data:`sys.platlibdir`: platform library directory name, set at configure time - by ``--with-platlibdir``, overrideable by the ``PYTHONPLATLIBDIR`` - environment variable. - - .. versionadded:: 3.9 + Part of the :ref:`Path Configuration ` output. .. c:member:: int buffered_stdio - If equals to 0, enable unbuffered mode, making the stdout and stderr - streams unbuffered. + If equals to 0 and :c:member:`~PyConfig.configure_c_stdio` is non-zero, + disable buffering on the C streams stdout and stderr. + + Set to 0 by the :option:`-u` command line option and the + :envvar:`PYTHONUNBUFFERED` environment variable. stdin is always opened in buffered mode. + Default: ``1``. + .. c:member:: int bytes_warning If equals to 1, issue a warning when comparing :class:`bytes` or :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with - :class:`int`. If equal or greater to 2, raise a :exc:`BytesWarning` - exception. + :class:`int`. + + If equal or greater to 2, raise a :exc:`BytesWarning` exception in these + cases. + + Incremented by the :option:`-b` command line option. + + Default: ``0``. .. c:member:: wchar_t* check_hash_pycs_mode - Control the validation behavior of hash-based ``.pyc`` files (see - :pep:`552`): :option:`--check-hash-based-pycs` command line option value. + Control the validation behavior of hash-based ``.pyc`` files: + value of the :option:`--check-hash-based-pycs` command line option. - Valid values: ``always``, ``never`` and ``default``. + Valid values: - The default value is: ``default``. + - ``L"always"``: Hash the source file for invalidation regardless of + value of the 'check_source' flag. + - ``L"never"``: Assume that hash-based pycs always are valid. + - ``L"default"``: The 'check_source' flag in hash-based pycs + determines invalidation. + + Default: ``L"default"``. + + See also :pep:`552` "Deterministic pycs". .. c:member:: int configure_c_stdio - If non-zero, configure C standard streams (``stdio``, ``stdout``, - ``stdout``). For example, set their mode to ``O_BINARY`` on Windows. + If non-zero, configure C standard streams: + + * On Windows, set the binary mode (``O_BINARY``) on stdin, stdout and + stderr. + * If :c:member:`~PyConfig.buffered_stdio` equals zero, disable buffering + of stdin, stdout and stderr streams. + * If :c:member:`~PyConfig.interactive` is non-zero, enable stream + buffering on stdin and stdout (only stdout on Windows). + + Default: ``1`` in Python config, ``0`` in isolated config. .. c:member:: int dev_mode If non-zero, enable the :ref:`Python Development Mode `. + Default: ``-1`` in Python mode, ``0`` in isolated mode. + .. c:member:: int dump_refs + Dump Python refererences? + If non-zero, dump all objects which are still alive at exit. - ``Py_TRACE_REFS`` macro must be defined in build. + Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. + + Need a special build of Python with the ``Py_TRACE_REFS`` macro defined. + + Default: ``0``. .. c:member:: wchar_t* exec_prefix - :data:`sys.exec_prefix`. + The site-specific directory prefix where the platform-dependent Python + files are installed: :data:`sys.exec_prefix`. + + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` output. .. c:member:: wchar_t* executable + The absolute path of the executable binary for the Python interpreter: :data:`sys.executable`. + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` output. + .. c:member:: int faulthandler + Enable faulthandler? + If non-zero, call :func:`faulthandler.enable` at startup. + Set to ``1`` by :option:`-X faulthandler <-X>` and the + :envvar:`PYTHONFAULTHANDLER` environment variable. + + Default: ``-1`` in Python mode, ``0`` in isolated mode. + .. c:member:: wchar_t* filesystem_encoding - Filesystem encoding: :func:`sys.getfilesystemencoding`. + :term:`Filesystem encoding `: + :func:`sys.getfilesystemencoding`. On macOS, Android and VxWorks: use ``"utf-8"`` by default. @@ -521,7 +639,7 @@ PyConfig ``mbstowcs()`` function decodes from a different encoding (usually Latin1). * ``"utf-8"`` if ``nl_langinfo(CODESET)`` returns an empty string. - * Otherwise, use the LC_CTYPE locale encoding: + * Otherwise, use the :term:`locale encoding`: ``nl_langinfo(CODESET)`` result. At Python statup, the encoding name is normalized to the Python codec @@ -531,7 +649,8 @@ PyConfig .. c:member:: wchar_t* filesystem_errors - Filesystem error handler: :func:`sys.getfilesystemencodeerrors`. + :term:`Filesystem error handler `: + :func:`sys.getfilesystemencodeerrors`. On Windows: use ``"surrogatepass"`` by default, or ``"replace"`` if :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of @@ -553,30 +672,62 @@ PyConfig Randomized hash function seed. If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly - at Pythonstartup, and :c:member:`~PyConfig.hash_seed` is ignored. + at Python startup, and :c:member:`~PyConfig.hash_seed` is ignored. + + Set by the :envvar:`PYTHONHASHSEED` environment variable. + + Default *use_hash_seed* value: ``-1`` in Python mode, ``0`` in isolated + mode. .. c:member:: wchar_t* home Python home directory. - Initialized from :envvar:`PYTHONHOME` environment variable value by - default. + If :c:func:`Py_SetPythonHome` has been called, use its argument if it is + not ``NULL``. + + Set by the :envvar:`PYTHONHOME` environment variable. + + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` input. .. c:member:: int import_time If non-zero, profile import time. + Set the ``1`` by the :option:`-X importtime <-X>` option and the + :envvar:`PYTHONPROFILEIMPORTTIME` environment variable. + + Default: ``0``. + .. c:member:: int inspect Enter interactive mode after executing a script or a command. + If greater than 0, enable inspect: when a script is passed as first + argument or the -c option is used, enter interactive mode after executing + the script or the command, even when :data:`sys.stdin` does not appear to + be a terminal. + + Incremented by the :option:`-i` command line option. Set to ``1`` if the + :envvar:`PYTHONINSPECT` environment variable is non-empty. + + Default: ``0``. + .. c:member:: int install_signal_handlers - Install signal handlers? + Install Python signal handlers? + + Default: ``1`` in Python mode, ``0`` in isolated mode. .. c:member:: int interactive - Interactive mode. + If greater than 0, enable the interactive mode (REPL). + + Incremented by the :option:`-i` command line option. + + Default: ``0``. .. c:member:: int isolated @@ -590,50 +741,93 @@ PyConfig * Set :c:member:`~PyConfig.use_environment` and :c:member:`~PyConfig.user_site_directory` to 0. + Default: ``0`` in Python mode, ``1`` in isolated mode. + + See also :c:member:`PyPreConfig.isolated`. + .. c:member:: int legacy_windows_stdio If non-zero, use :class:`io.FileIO` instead of :class:`io.WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` and :data:`sys.stderr`. + Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment + variable is set to a non-empty string. + Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for Windows specific code. + Default: ``0``. + + See also the :pep:`528` (Change Windows console encoding to UTF-8). + .. c:member:: int malloc_stats If non-zero, dump statistics on :ref:`Python pymalloc memory allocator ` at exit. + Set to ``1`` by the :envvar:`PYTHONMALLOCSTATS` environment variable. + The option is ignored if Python is built using ``--without-pymalloc``. + Default: ``0``. + + .. c:member:: wchar_t* platlibdir + + Platform library directory name: :data:`sys.platlibdir`. + + Set by the :envvar:`PYTHONPLATLIBDIR` environment variable. + + Default: value of the ``PLATLIBDIR`` macro which is set at configure time + by ``--with-platlibdir`` (default: ``"lib"``). + + Part of the :ref:`Path Configuration ` input. + + .. versionadded:: 3.9 + .. c:member:: wchar_t* pythonpath_env - Module search paths as a string separated by ``DELIM`` + Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` (:data:`os.path.pathsep`). - Initialized from :envvar:`PYTHONPATH` environment variable value by - default. + Set by the :envvar:`PYTHONPATH` environment variable. + + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` input. .. c:member:: PyWideStringList module_search_paths .. c:member:: int module_search_paths_set - :data:`sys.path`. If :c:member:`~PyConfig.module_search_paths_set` is - equal to 0, the :c:member:`~PyConfig.module_search_paths` is overridden - by the function calculating the :ref:`Path Configuration - `. + Module search paths: :data:`sys.path`. + + If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the + function calculating the :ref:`Path Configuration ` + overrides the :c:member:`~PyConfig.module_search_paths` and sets + :c:member:`~PyConfig.module_search_paths_set` to ``1``. + + Default: empty list (``module_search_paths``) and ``0`` + (``module_search_paths_set``). + + Part of the :ref:`Path Configuration ` output. .. c:member:: int optimization_level Compilation optimization level: - * 0: Peephole optimizer (and ``__debug__`` is set to ``True``) - * 1: Remove assertions, set ``__debug__`` to ``False`` - * 2: Strip docstrings + * ``0``: Peephole optimizer, set ``__debug__`` to ``True``. + * ``1``: Level 0, remove assertions, set ``__debug__`` to ``False``. + * ``2``: Level 1, strip docstrings. + + Incremented by the :option:`-O` command line option. Set to the + :envvar:`PYTHONOPTIMIZE` environment variable value. + + Default: ``0``. .. c:member:: PyWideStringList orig_argv The list of the original command line arguments passed to the Python - executable. + executable: :data:`sys.orig_argv`. If :c:member:`~PyConfig.orig_argv` list is empty and :c:member:`~PyConfig.argv` is not a list only containing an empty @@ -645,57 +839,117 @@ PyConfig See also the :c:member:`~PyConfig.argv` member and the :c:func:`Py_GetArgcArgv` function. + Default: empty list. + .. versionadded:: 3.10 .. c:member:: int parse_argv + Parse command line arguments? + If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular - Python command line arguments, and strip Python arguments from - :c:member:`~PyConfig.argv`: see :ref:`Command Line Arguments - `. + Python parses :ref:`command line arguments `, and strip + Python arguments from :c:member:`~PyConfig.argv`. + + Default: ``1`` in Python mode, ``0`` in isolated mode. .. c:member:: int parser_debug - If non-zero, turn on parser debugging output (for expert only, depending + Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending on compilation options). + Incremented by the :option:`-d` command line option. Set to the + :envvar:`PYTHONDEBUG` environment variable value. + + Default: ``0``. + .. c:member:: int pathconfig_warnings - If equal to 0, suppress warnings when calculating the :ref:`Path - Configuration ` (Unix only, Windows does not log any - warning). Otherwise, warnings are written into ``stderr``. + On Unix, if non-zero, calculating the :ref:`Path Configuration + ` can log warnings into ``stderr``. If equals to 0, + suppress these warnings. + + It has no effect on Windows. + + Default: ``1`` in Python mode, ``0`` in isolated mode. + + Part of the :ref:`Path Configuration ` input. .. c:member:: wchar_t* prefix - :data:`sys.prefix`. + The site-specific directory prefix where the platform independent Python + files are installed: :data:`sys.prefix`. + + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` output. .. c:member:: wchar_t* program_name - Program name. Used to initialize :c:member:`~PyConfig.executable`, and in - early error messages. + Program name used to initialize :c:member:`~PyConfig.executable` and in + early error messages during Python initialization. + + * If :func:`Py_SetProgramName` has been called, use its argument. + * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. + * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use + :envvar:`__PYVENV_LAUNCHER__` environment variable if set. + * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and + non-empty. + * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other + platforms. + + Default: ``NULL``. + + Part of the :ref:`Path Configuration ` input. .. c:member:: wchar_t* pycache_prefix - :data:`sys.pycache_prefix`: ``.pyc`` cache prefix. + Directory where cached ``.pyc`` files are written: + :data:`sys.pycache_prefix`. + + Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and + the :envvar:`PYTHONPYCACHEPREFIX` environment variable. If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. + Default: ``NULL``. + .. c:member:: int quiet - Quiet mode. For example, don't display the copyright and version messages - in interactive mode. + Quiet mode. If greater than 0, don't display the copyright and version at + Python startup in interactive mode. + + Incremented by the :option:`-q` command line option. + + Default: ``0``. .. c:member:: wchar_t* run_command - ``python3 -c COMMAND`` argument. Used by :c:func:`Py_RunMain`. + Value of the :option:`-c` command line option. + + Used by :c:func:`Py_RunMain`. + + Default: ``NULL``. .. c:member:: wchar_t* run_filename - ``python3 FILENAME`` argument. Used by :c:func:`Py_RunMain`. + Filename passed on the command line: trailing command line argument + without :option:`-c` or :option:`-m`. + + For example, it is set to ``script.py`` by the ``python3 script.py arg`` + command. + + Used by :c:func:`Py_RunMain`. + + Default: ``NULL``. .. c:member:: wchar_t* run_module - ``python3 -m MODULE`` argument. Used by :c:func:`Py_RunMain`. + Value of the :option:`-m` command line option. + + Used by :c:func:`Py_RunMain`. + + Default: ``NULL``. .. c:member:: int show_ref_count @@ -705,64 +959,146 @@ PyConfig Need a debug build of Python (``Py_REF_DEBUG`` macro must be defined). + Default: ``0``. + .. c:member:: int site_import Import the :mod:`site` module at startup? + If equal to zero, disable the import of the module site and the + site-dependent manipulations of :data:`sys.path` that it entails. + + Also disable these manipulations if the :mod:`site` module is explicitly + imported later (call :func:`site.main` if you want them to be triggered). + + Set to ``0`` by the :option:`-S` command line option. + + :data:`sys.flags.no_site` is set to the inverted value of + :c:member:`~PyConfig.site_import`. + + Default: ``1``. + .. c:member:: int skip_source_first_line - Skip the first line of the source? + If non-zero, skip the first line of the :c:member:`PyConfig.run_filename` + source. + + It allows the usage of non-Unix forms of ``#!cmd``. This is intended for + a DOS specific hack only. + + Set to ``1`` by the :option:`-x` command line option. + + Default: ``0``. .. c:member:: wchar_t* stdio_encoding .. c:member:: wchar_t* stdio_errors Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and - :data:`sys.stderr`. + :data:`sys.stderr` (but :data:`sys.stderr` always uses + ``"backslashreplace"`` error handler). + + If :c:func:`Py_SetStandardStreamEncoding` has been called, use its + *error* and *errors* arguments if they are not ``NULL``. + + Use the :envvar:`PYTHONIOENCODING` environment variable if it is + non-empty. + + Default encoding: + + * ``"UTF-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. + * Otherwise, use the :term:`locale encoding`. + + Default error handler: + + * On Windows: use ``"surrogateescape"``. + * ``"surrogateescape"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero, + or if the LC_CTYPE locale is "C" or "POSIX". + * ``"strict"`` otherwise. .. c:member:: int tracemalloc + Enable tracemalloc? + If non-zero, call :func:`tracemalloc.start` at startup. + Set by :option:`-X tracemalloc=N <-X>` command line option and by the + :envvar:`PYTHONTRACEMALLOC` environment variable. + + Default: ``-1`` in Python mode, ``0`` in isolated mode. + .. c:member:: int use_environment - If greater than 0, use :ref:`environment variables `. + Use :ref:`environment variables `? + + If equals to zero, ignore the :ref:`environment variables + `. + + Default: ``1`` in Python config and ``0`` in isolated config. .. c:member:: int user_site_directory - If non-zero, add user site directory to :data:`sys.path`. + If non-zero, add the user site directory to :data:`sys.path`. + + Set to ``0`` by the :option:`-s` and :option:`-I` command line options. + + Set to ``0`` by the :envvar:`PYTHONNOUSERSITE` environment variable. + + Default: ``1`` in Python mode, ``0`` in isolated mode. .. c:member:: int verbose - If non-zero, enable verbose mode. + Verbose mode. If greater than 0, print a message each time a module is + imported, showing the place (filename or built-in module) from which + it is loaded. + + If greater or equal to 2, print a message for each file that is checked + for when searching for a module. Also provides information on module + cleanup at exit. + + Incremented by the :option:`-v` command line option. + + Set to the :envvar:`PYTHONVERBOSE` environment variable value. + + Default: ``0``. .. c:member:: PyWideStringList warnoptions - :data:`sys.warnoptions`: options of the :mod:`warnings` module to build - warnings filters: lowest to highest priority. + Options of the :mod:`warnings` module to build warnings filters, lowest + to highest priority: :data:`sys.warnoptions`. The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse order: the last :c:member:`PyConfig.warnoptions` item becomes the first item of :data:`warnings.filters` which is checked first (highest priority). + Default: empty list. + .. c:member:: int write_bytecode - If non-zero, write ``.pyc`` files. + If equal to 0, Python won't try to write ``.pyc`` files on the import of + source modules. + + Set to ``0`` by the :option:`-B` command line option and the + :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. :data:`sys.dont_write_bytecode` is initialized to the inverted value of :c:member:`~PyConfig.write_bytecode`. + Default: ``1``. + .. c:member:: PyWideStringList xoptions - :data:`sys._xoptions`. + Values of the :option:`-X` command line options: :data:`sys._xoptions`. -If ``parse_argv`` is non-zero, ``argv`` arguments are parsed the same -way the regular Python parses command line arguments, and Python -arguments are stripped from ``argv``: see :ref:`Command Line Arguments -`. + Default: empty list. -The ``xoptions`` options are parsed to set other options: see :option:`-X` -option. +If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` +arguments are parsed the same way the regular Python parses :ref:`command line +arguments `, and Python arguments are stripped from +:c:member:`~PyConfig.argv`. + +The :c:member:`~PyConfig.xoptions` options are parsed to set other options: see +the :option:`-X` command line option. .. versionchanged:: 3.9 @@ -781,9 +1117,9 @@ Function to initialize Python: The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. -If ``PyImport_FrozenModules``, ``PyImport_AppendInittab()`` or -``PyImport_ExtendInittab()`` are used, they must be set or called after Python -preinitialization and before the Python initialization. +If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or +:c:func:`PyImport_ExtendInittab` are used, they must be set or called after +Python preinitialization and before the Python initialization. Example setting the program name:: @@ -828,7 +1164,7 @@ configuration, and then override some parameters:: Implicitly preinitialize Python. */ status = PyConfig_SetBytesString(&config, &config.program_name, - program_name); + program_name); if (PyStatus_Exception(status)) { goto done; } @@ -894,7 +1230,8 @@ the regular Python. Environments variables and command line arguments are used to configure Python, whereas global configuration variables are ignored. -This function enables C locale coercion (:pep:`538`) and UTF-8 Mode +This function enables C locale coercion (:pep:`538`) +and :ref:`Python UTF-8 Mode ` (:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and :envvar:`PYTHONCOERCECLOCALE` environment variables. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst index 9ac91790978..97717f5fc19 100644 --- a/Doc/c-api/sys.rst +++ b/Doc/c-api/sys.rst @@ -118,22 +118,21 @@ Operating System Utilities .. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) - Decode a byte string from the locale encoding with the :ref:`surrogateescape - error handler `: undecodable bytes are decoded as - characters in range U+DC80..U+DCFF. If a byte sequence can be decoded as a - surrogate character, escape the bytes using the surrogateescape error - handler instead of decoding them. + .. warning:: + This function should not be called directly: use the :c:type:`PyConfig` + API with the :c:func:`PyConfig_SetBytesString` function which ensures + that :ref:`Python is preinitialized `. - Encoding, highest priority to lowest priority: + This function must not be called before :ref:`Python is preinitialized + ` and so that the LC_CTYPE locale is properly configured: see + the :c:func:`Py_PreInitialize` function. - * ``UTF-8`` on macOS, Android, and VxWorks; - * ``UTF-8`` on Windows if :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero; - * ``UTF-8`` if the Python UTF-8 mode is enabled; - * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, - ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), - and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the - ``ISO-8859-1`` encoding. - * the current locale encoding. + Decode a byte string from the :term:`filesystem encoding and error handler`. + If the error handler is :ref:`surrogateescape error handler + `, undecodable bytes are decoded as characters in range + U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate + character, the bytes are escaped using the surrogateescape error handler + instead of decoding them. Return a pointer to a newly allocated wide character string, use :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write @@ -143,6 +142,10 @@ Operating System Utilities not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to ``(size_t)-2`` on decoding error. + The :term:`filesystem encoding and error handler` are selected by + :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + Decoding errors should never happen, unless there is a bug in the C library. @@ -157,7 +160,8 @@ Operating System Utilities .. versionadded:: 3.5 .. versionchanged:: 3.7 - The function now uses the UTF-8 encoding in the UTF-8 mode. + The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode + `. .. versionchanged:: 3.8 The function now uses the UTF-8 encoding on Windows if @@ -166,22 +170,10 @@ Operating System Utilities .. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) - Encode a wide character string to the locale encoding with the - :ref:`surrogateescape error handler `: surrogate characters - in the range U+DC80..U+DCFF are converted to bytes 0x80..0xFF. - - Encoding, highest priority to lowest priority: - - * ``UTF-8`` on macOS, Android, and VxWorks; - * ``UTF-8`` on Windows if :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero; - * ``UTF-8`` if the Python UTF-8 mode is enabled; - * ``ASCII`` if the ``LC_CTYPE`` locale is ``"C"``, - ``nl_langinfo(CODESET)`` returns the ``ASCII`` encoding (or an alias), - and :c:func:`mbstowcs` and :c:func:`wcstombs` functions uses the - ``ISO-8859-1`` encoding. - * the current locale encoding. - - The function uses the UTF-8 encoding in the Python UTF-8 mode. + Encode a wide character string to the :term:`filesystem encoding and error + handler`. If the error handler is :ref:`surrogateescape error handler + `, surrogate characters in the range U+DC80..U+DCFF are + converted to bytes 0x80..0xFF. Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` to free the memory. Return ``NULL`` on encoding error or memory allocation @@ -190,9 +182,18 @@ Operating System Utilities If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on success, or set to the index of the invalid character on encoding error. + The :term:`filesystem encoding and error handler` are selected by + :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back to a wide character string. + .. warning:: + This function must not be called before :ref:`Python is preinitialized + ` and so that the LC_CTYPE locale is properly configured: see + the :c:func:`Py_PreInitialize` function. + .. seealso:: The :c:func:`PyUnicode_EncodeFSDefault` and @@ -201,7 +202,8 @@ Operating System Utilities .. versionadded:: 3.5 .. versionchanged:: 3.7 - The function now uses the UTF-8 encoding in the UTF-8 mode. + The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode + `. .. versionchanged:: 3.8 The function now uses the UTF-8 encoding on Windows if diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst index 54bd0a3cbb6..b7f99d32558 100644 --- a/Doc/c-api/unicode.rst +++ b/Doc/c-api/unicode.rst @@ -783,7 +783,7 @@ system. :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at Python startup). - This function ignores the Python UTF-8 mode. + This function ignores the :ref:`Python UTF-8 Mode `. .. seealso:: @@ -819,7 +819,7 @@ system. :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at Python startup). - This function ignores the Python UTF-8 mode. + This function ignores the :ref:`Python UTF-8 Mode `. .. seealso:: @@ -878,8 +878,7 @@ conversion function: .. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) - Decode a string using :c:data:`Py_FileSystemDefaultEncoding` and the - :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. + Decode a string from the :term:`filesystem encoding and error handler`. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. @@ -899,8 +898,8 @@ conversion function: .. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) - Decode a null-terminated string using :c:data:`Py_FileSystemDefaultEncoding` - and the :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. + Decode a null-terminated string from the :term:`filesystem encoding and + error handler`. If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the locale encoding. diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst index b908cb8354f..0f760eaa7ad 100644 --- a/Doc/c-api/veryhigh.rst +++ b/Doc/c-api/veryhigh.rst @@ -112,9 +112,9 @@ the same library that the Python runtime is using. Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read from *fp* instead of an in-memory string. *filename* should be the name of - the file, it is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). If *closeit* is true, the file is - closed before PyRun_SimpleFileExFlags returns. + the file, it is decoded from :term:`filesystem encoding and error handler`. + If *closeit* is true, the file is closed before + ``PyRun_SimpleFileExFlags()`` returns. .. note:: On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``). @@ -132,7 +132,7 @@ the same library that the Python runtime is using. Read and execute a single statement from a file associated with an interactive device according to the *flags* argument. The user will be prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - filesystem encoding (:func:`sys.getfilesystemencoding`). + :term:`filesystem encoding and error handler`. Returns ``0`` when the input was executed successfully, ``-1`` if there was an exception, or an error code @@ -151,9 +151,8 @@ the same library that the Python runtime is using. Read and execute statements from a file associated with an interactive device until EOF is reached. The user will be prompted using ``sys.ps1`` and - ``sys.ps2``. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). Returns ``0`` at EOF or a negative - number upon failure. + ``sys.ps2``. *filename* is decoded from the :term:`filesystem encoding and + error handler`. Returns ``0`` at EOF or a negative number upon failure. .. c:var:: int (*PyOS_InputHook)(void) @@ -206,8 +205,8 @@ the same library that the Python runtime is using. Parse Python source code from *str* using the start token *start* according to the *flags* argument. The result can be used to create a code object which can be evaluated efficiently. This is useful if a code fragment must be evaluated - many times. *filename* is decoded from the filesystem encoding - (:func:`sys.getfilesystemencoding`). + many times. *filename* is decoded from the :term:`filesystem encoding and + error handler`. .. c:function:: struct _node* PyParser_SimpleParseFile(FILE *fp, const char *filename, int start) @@ -262,7 +261,7 @@ the same library that the Python runtime is using. Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from *fp* instead of an in-memory string. *filename* should be the name of the file, - it is decoded from the filesystem encoding (:func:`sys.getfilesystemencoding`). + it is decoded from the :term:`filesystem encoding and error handler`. If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags` returns. @@ -301,7 +300,7 @@ the same library that the Python runtime is using. .. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) Like :c:func:`Py_CompileStringObject`, but *filename* is a byte string - decoded from the filesystem encoding (:func:`os.fsdecode`). + decoded from the :term:`filesystem encoding and error handler`. .. versionadded:: 3.2 diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 4fd01e0160c..506973e964b 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -386,6 +386,25 @@ Glossary file-like object A synonym for :term:`file object`. + filesystem encoding and error handler + Encoding and error handler used by Python to decode bytes from the + operating system and encode Unicode to the operating system. + + The filesystem encoding must guarantee to successfully decode all bytes + below 128. If the file system encoding fails to provide this guarantee, + API functions can raise :exc:`UnicodeError`. + + The :func:`sys.getfilesystemencoding` and + :func:`sys.getfilesystemencodeerrors` functions can be used to get the + filesystem encoding and error handler. + + The :term:`filesystem encoding and error handler` are configured at + Python startup by the :c:func:`PyConfig_Read` function: see + :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. + + See also the :term:`locale encoding`. + finder An object that tries to find the :term:`loader` for a module that is being imported. @@ -673,6 +692,18 @@ Glossary thread removes *key* from *mapping* after the test, but before the lookup. This issue can be solved with locks or by using the EAFP approach. + locale encoding + On Unix, it is the encoding of the LC_CTYPE locale. It can be set with + ``locale.setlocale(locale.LC_CTYPE, new_locale)``. + + On Windows, it is is the ANSI code page (ex: ``cp1252``). + + ``locale.getpreferredencoding(False)`` can be used to get the locale + encoding. + + Python uses the :term:`filesystem encoding and error handler` to convert + between Unicode filenames and bytes filenames. + list A built-in Python :term:`sequence`. Despite its name it is more akin to an array in other languages than to a linked list since access to diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst index e948c1e3c66..535b21bd4a5 100644 --- a/Doc/howto/unicode.rst +++ b/Doc/howto/unicode.rst @@ -609,9 +609,9 @@ implemented by converting the Unicode string into some encoding that varies depending on the system. Today Python is converging on using UTF-8: Python on MacOS has used UTF-8 for several versions, and Python 3.6 switched to using UTF-8 on Windows as well. On Unix systems, -there will only be a filesystem encoding if you've set the ``LANG`` or -``LC_CTYPE`` environment variables; if you haven't, the default -encoding is again UTF-8. +there will only be a :term:`filesystem encoding `. if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if +you haven't, the default encoding is again UTF-8. The :func:`sys.getfilesystemencoding` function returns the encoding to use on your current system, in case you want to do the encoding manually, but there's @@ -633,8 +633,8 @@ provided the directory path as bytes or a Unicode string. If you pass a Unicode string as the path, filenames will be decoded using the filesystem's encoding and a list of Unicode strings will be returned, while passing a byte path will return the filenames as bytes. For example, -assuming the default filesystem encoding is UTF-8, running the following -program:: +assuming the default :term:`filesystem encoding ` is UTF-8, running the following program:: fn = 'filename\u4500abc' f = open(fn, 'w') diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst index d5a40cdeeac..e6ed59496c2 100644 --- a/Doc/library/devmode.rst +++ b/Doc/library/devmode.rst @@ -93,6 +93,9 @@ The Python Development Mode does not prevent the :option:`-O` command line option from removing :keyword:`assert` statements nor from setting :const:`__debug__` to ``False``. +The Python Development Mode can only be enabled at the Python startup. Its +value can be read from :data:`sys.flags.dev_mode `. + .. versionchanged:: 3.8 The :class:`io.IOBase` destructor now logs ``close()`` exceptions. diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst index df2cda9d67a..8fb25a50e2d 100644 --- a/Doc/library/exceptions.rst +++ b/Doc/library/exceptions.rst @@ -313,8 +313,8 @@ The following exceptions are the exceptions that are usually raised. .. versionchanged:: 3.4 The :attr:`filename` attribute is now the original file name passed to the function, instead of the name encoded to or decoded from the - filesystem encoding. Also, the *filename2* constructor argument and - attribute was added. + :term:`filesystem encoding and error handler`. Also, the *filename2* + constructor argument and attribute was added. .. exception:: OverflowError diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst index 678148a0dda..0a77be47fb8 100644 --- a/Doc/library/locale.rst +++ b/Doc/library/locale.rst @@ -315,21 +315,25 @@ The :mod:`locale` module defines the following exception and functions: .. function:: getpreferredencoding(do_setlocale=True) - Return the encoding used for text data, according to user preferences. User - preferences are expressed differently on different systems, and might not be - available programmatically on some systems, so this function only returns a - guess. + Return the :term:`locale encoding` used for text data, according to user + preferences. User preferences are expressed differently on different + systems, and might not be available programmatically on some systems, so + this function only returns a guess. - On some systems, it is necessary to invoke :func:`setlocale` to obtain the user - preferences, so this function is not thread-safe. If invoking setlocale is not - necessary or desired, *do_setlocale* should be set to ``False``. + On some systems, it is necessary to invoke :func:`setlocale` to obtain the + user preferences, so this function is not thread-safe. If invoking setlocale + is not necessary or desired, *do_setlocale* should be set to ``False``. - On Android or in the UTF-8 mode (:option:`-X` ``utf8`` option), always - return ``'UTF-8'``, the locale and the *do_setlocale* argument are ignored. + On Android or if the :ref:`Python UTF-8 Mode ` is enabled, always + return ``'UTF-8'``, the :term:`locale encoding` and the *do_setlocale* + argument are ignored. + + The :ref:`Python preinitialization ` configures the LC_CTYPE + locale. See also the :term:`filesystem encoding and error handler`. .. versionchanged:: 3.7 - The function now always returns ``UTF-8`` on Android or if the UTF-8 mode - is enabled. + The function now always returns ``UTF-8`` on Android or if the + :ref:`Python UTF-8 Mode ` is enabled. .. function:: normalize(localename) diff --git a/Doc/library/os.rst b/Doc/library/os.rst index 718d98138d2..f9f35b31243 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -68,8 +68,13 @@ File Names, Command Line Arguments, and Environment Variables In Python, file names, command line arguments, and environment variables are represented using the string type. On some systems, decoding these strings to and from bytes is necessary before passing them to the operating system. Python -uses the file system encoding to perform this conversion (see -:func:`sys.getfilesystemencoding`). +uses the :term:`filesystem encoding and error handler` to perform this +conversion (see :func:`sys.getfilesystemencoding`). + +The :term:`filesystem encoding and error handler` are configured at Python +startup by the :c:func:`PyConfig_Read` function: see +:c:member:`~PyConfig.filesystem_encoding` and +:c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. .. versionchanged:: 3.1 On some systems, conversion using the file system encoding may fail. In this @@ -79,9 +84,70 @@ uses the file system encoding to perform this conversion (see original byte on encoding. -The file system encoding must guarantee to successfully decode all bytes -below 128. If the file system encoding fails to provide this guarantee, API -functions may raise UnicodeErrors. +The :term:`file system encoding ` must +guarantee to successfully decode all bytes below 128. If the file system +encoding fails to provide this guarantee, API functions can raise +:exc:`UnicodeError`. + +See also the :term:`locale encoding`. + + +.. _utf8-mode: + +Python UTF-8 Mode +----------------- + +.. versionadded:: 3.7 + See :pep:`540` for more details. + +The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage +of the UTF-8 encoding: + +* Use UTF-8 as the :term:`filesystem encoding `. +* :func:`sys.getfilesystemencoding()` returns ``'UTF-8'``. +* :func:`locale.getpreferredencoding()` returns ``'UTF-8'`` (the *do_setlocale* + argument has no effect). +* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use + UTF-8 as their text encoding, with the ``surrogateescape`` + :ref:`error handler ` being enabled for :data:`sys.stdin` + and :data:`sys.stdout` (:data:`sys.stderr` continues to use + ``backslashreplace`` as it does in the default locale-aware mode) + +Note that the standard stream settings in UTF-8 mode can be overridden by +:envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware +mode). + +As a consequence of the changes in those lower level APIs, other higher +level APIs also exhibit different default behaviours: + +* Command line arguments, environment variables and filenames are decoded + to text using the UTF-8 encoding. +* :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. +* :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 + encoding by default. However, they still use the strict error handler by + default so that attempting to open a binary file in text mode is likely + to raise an exception rather than producing nonsense data. + +The :ref:`Python UTF-8 Mode ` is enabled if the LC_CTYPE locale is +``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read` +function). + +It can be enabled or disabled using the :option:`-X utf8 <-X>` command line +option and the :envvar:`PYTHONUTF8` environment variable. + +If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the +interpreter defaults to using the current locale settings, *unless* the current +locale is identified as a legacy ASCII-based locale (as described for +:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or +fails. In such legacy locales, the interpreter will default to enabling UTF-8 +mode unless explicitly instructed not to do so. + +The Python UTF-8 Mode can only be enabled at the Python startup. Its value +can be read from :data:`sys.flags.utf8_mode `. + +See also the :ref:`UTF-8 mode on Windows ` +and the :term:`filesystem encoding and error handler`. .. _os-procinfo: @@ -165,9 +231,9 @@ process and user. .. function:: fsencode(filename) - Encode :term:`path-like ` *filename* to the filesystem - encoding with ``'surrogateescape'`` error handler, or ``'strict'`` on - Windows; return :class:`bytes` unchanged. + Encode :term:`path-like ` *filename* to the + :term:`filesystem encoding and error handler`; return :class:`bytes` + unchanged. :func:`fsdecode` is the reverse function. @@ -181,8 +247,8 @@ process and user. .. function:: fsdecode(filename) Decode the :term:`path-like ` *filename* from the - filesystem encoding with ``'surrogateescape'`` error handler, or ``'strict'`` - on Windows; return :class:`str` unchanged. + :term:`filesystem encoding and error handler`; return :class:`str` + unchanged. :func:`fsencode` is the reverse function. @@ -3246,7 +3312,7 @@ These functions are all available on Linux only. Removes the extended filesystem attribute *attribute* from *path*. *attribute* should be bytes or str (directly or indirectly through the :class:`PathLike` interface). If it is a string, it is encoded - with the filesystem encoding. + with the :term:`filesystem encoding and error handler`. This function can support :ref:`specifying a file descriptor ` and :ref:`not following symlinks `. @@ -3262,7 +3328,7 @@ These functions are all available on Linux only. Set the extended filesystem attribute *attribute* on *path* to *value*. *attribute* must be a bytes or str with no embedded NULs (directly or indirectly through the :class:`PathLike` interface). If it is a str, - it is encoded with the filesystem encoding. *flags* may be + it is encoded with the :term:`filesystem encoding and error handler`. *flags* may be :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is given and the attribute does not exist, ``EEXISTS`` will be raised. If :data:`XATTR_CREATE` is given and the attribute already exists, the diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index f0acfcfe639..0f13adcf0e5 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -627,21 +627,24 @@ always available. .. function:: getfilesystemencoding() - Return the name of the encoding used to convert between Unicode - filenames and bytes filenames. + Get the :term:`filesystem encoding `: + the encoding used with the :term:`filesystem error handler ` to convert between Unicode filenames and bytes + filenames. The filesystem error handler is returned from + :func:`getfilesystemencoding`. For best compatibility, str should be used for filenames in all cases, although representing filenames as bytes is also supported. Functions accepting or returning filenames should support either str or bytes and internally convert to the system's preferred representation. - This encoding is always ASCII-compatible. - :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that the correct encoding and errors mode are used. - The filesystem encoding is initialized from - :c:member:`PyConfig.filesystem_encoding`. + The :term:`filesystem encoding and error handler` are configured at Python + startup by the :c:func:`PyConfig_Read` function: see + :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. .. versionchanged:: 3.2 :func:`getfilesystemencoding` result cannot be ``None`` anymore. @@ -651,20 +654,25 @@ always available. and :func:`_enablelegacywindowsfsencoding` for more information. .. versionchanged:: 3.7 - Return 'utf-8' in the UTF-8 mode. + Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is + enabled. .. function:: getfilesystemencodeerrors() - Return the name of the error mode used to convert between Unicode filenames - and bytes filenames. The encoding name is returned from + Get the :term:`filesystem error handler `: the error handler used with the :term:`filesystem encoding + ` to convert between Unicode + filenames and bytes filenames. The filesystem encoding is returned from :func:`getfilesystemencoding`. :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that the correct encoding and errors mode are used. - The filesystem error handler is initialized from - :c:member:`PyConfig.filesystem_errors`. + The :term:`filesystem encoding and error handler` are configured at Python + startup by the :c:func:`PyConfig_Read` function: see + :c:member:`~PyConfig.filesystem_encoding` and + :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. .. versionadded:: 3.6 @@ -1457,8 +1465,9 @@ always available. .. function:: _enablelegacywindowsfsencoding() - Changes the default filesystem encoding and errors mode to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to 3.6. + Changes the :term:`filesystem encoding and error handler` to 'mbcs' and + 'replace' respectively, for consistency with versions of Python prior to + 3.6. This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment variable before launching Python. @@ -1488,9 +1497,8 @@ always available. returned by the :func:`open` function. Their parameters are chosen as follows: - * The character encoding is platform-dependent. Non-Windows - platforms use the locale encoding (see - :meth:`locale.getpreferredencoding()`). + * The encoding and error handling are is initialized from + :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. On Windows, UTF-8 is used for the console device. Non-character devices such as disk files and pipes use the system locale @@ -1498,7 +1506,7 @@ always available. devices such as NUL (i.e. where ``isatty()`` returns ``True``) use the value of the console input and output codepages at startup, respectively for stdin and stdout/stderr. This defaults to the - system locale encoding if the process is not initially attached + system :term:`locale encoding` if the process is not initially attached to a console. The special behaviour of the console can be overridden diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst index 603b0e105fb..04e0f3267db 100644 --- a/Doc/using/cmdline.rst +++ b/Doc/using/cmdline.rst @@ -447,10 +447,9 @@ Miscellaneous options * ``-X dev``: enable :ref:`Python Development Mode `, introducing additional runtime checks that are too expensive to be enabled by default. - * ``-X utf8`` enables UTF-8 mode for operating system interfaces, overriding - the default locale-aware mode. ``-X utf8=0`` explicitly disables UTF-8 - mode (even when it would otherwise activate automatically). - See :envvar:`PYTHONUTF8` for more details. + * ``-X utf8`` enables the :ref:`Python UTF-8 Mode `. + ``-X utf8=0`` explicitly disables :ref:`Python UTF-8 Mode ` + (even when it would otherwise activate automatically). * ``-X pycache_prefix=PATH`` enables writing ``.pyc`` files to a parallel tree rooted at the given directory instead of to the code tree. See also :envvar:`PYTHONPYCACHEPREFIX`. @@ -810,9 +809,10 @@ conflict. .. envvar:: PYTHONLEGACYWINDOWSFSENCODING - If set to a non-empty string, the default filesystem encoding and errors mode - will revert to their pre-3.6 values of 'mbcs' and 'replace', respectively. - Otherwise, the new defaults 'utf-8' and 'surrogatepass' are used. + If set to a non-empty string, the default :term:`filesystem encoding and + error handler` mode will revert to their pre-3.6 values of 'mbcs' and + 'replace', respectively. Otherwise, the new defaults 'utf-8' and + 'surrogatepass' are used. This may also be enabled at runtime with :func:`sys._enablelegacywindowsfsencoding()`. @@ -898,54 +898,14 @@ conflict. .. envvar:: PYTHONUTF8 - If set to ``1``, enables the interpreter's UTF-8 mode, where ``UTF-8`` is - used as the text encoding for system interfaces, regardless of the - current locale setting. + If set to ``1``, enable the :ref:`Python UTF-8 Mode `. - This means that: - - * :func:`sys.getfilesystemencoding()` returns ``'UTF-8'`` (the locale - encoding is ignored). - * :func:`locale.getpreferredencoding()` returns ``'UTF-8'`` (the locale - encoding is ignored, and the function's ``do_setlocale`` parameter has no - effect). - * :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use - UTF-8 as their text encoding, with the ``surrogateescape`` - :ref:`error handler ` being enabled for :data:`sys.stdin` - and :data:`sys.stdout` (:data:`sys.stderr` continues to use - ``backslashreplace`` as it does in the default locale-aware mode) - - As a consequence of the changes in those lower level APIs, other higher - level APIs also exhibit different default behaviours: - - * Command line arguments, environment variables and filenames are decoded - to text using the UTF-8 encoding. - * :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. - * :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 - encoding by default. However, they still use the strict error handler by - default so that attempting to open a binary file in text mode is likely - to raise an exception rather than producing nonsense data. - - Note that the standard stream settings in UTF-8 mode can be overridden by - :envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware - mode). - - If set to ``0``, the interpreter runs in its default locale-aware mode. + If set to ``0``, disable the :ref:`Python UTF-8 Mode `. Setting any other non-empty string causes an error during interpreter initialisation. - If this environment variable is not set at all, then the interpreter defaults - to using the current locale settings, *unless* the current locale is - identified as a legacy ASCII-based locale - (as described for :envvar:`PYTHONCOERCECLOCALE`), and locale coercion is - either disabled or fails. In such legacy locales, the interpreter will - default to enabling UTF-8 mode unless explicitly instructed not to do so. - - Also available as the :option:`-X` ``utf8`` option. - .. versionadded:: 3.7 - See :pep:`540` for more details. Debug-mode variables diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 275495bc6d1..78c1e03f746 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -614,21 +614,14 @@ Page). Python uses it for the default encoding of text files (e.g. This may cause issues because UTF-8 is widely used on the internet and most Unix systems, including WSL (Windows Subsystem for Linux). -You can use UTF-8 mode to change the default text encoding to UTF-8. -You can enable UTF-8 mode via the ``-X utf8`` command line option, or -the ``PYTHONUTF8=1`` environment variable. See :envvar:`PYTHONUTF8` for -enabling UTF-8 mode, and :ref:`setting-envvars` for how to modify -environment variables. +You can use the :ref:`Python UTF-8 Mode ` to change the default text +encoding to UTF-8. You can enable the :ref:`Python UTF-8 Mode ` via +the ``-X utf8`` command line option, or the ``PYTHONUTF8=1`` environment +variable. See :envvar:`PYTHONUTF8` for enabling UTF-8 mode, and +:ref:`setting-envvars` for how to modify environment variables. -When UTF-8 mode is enabled: - -* :func:`locale.getpreferredencoding` returns ``'UTF-8'`` instead of - the system encoding. This function is used for the default text - encoding in many places, including :func:`open`, :class:`Popen`, - :meth:`Path.read_text`, etc. -* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` - all use UTF-8 as their text encoding. -* You can still use the system encoding via the "mbcs" codec. +When the :ref:`Python UTF-8 Mode ` is enabled, you can still use the +system encoding (the ANSI Code Page) via the "mbcs" codec. Note that adding ``PYTHONUTF8=1`` to the default environment variables will affect all Python 3.7+ applications on your system. @@ -641,7 +634,8 @@ temporarily or use the ``-X utf8`` command line option. on Windows for: * Console I/O including standard I/O (see :pep:`528` for details). - * The filesystem encoding (see :pep:`529` for details). + * The :term:`filesystem encoding ` + (see :pep:`529` for details). .. _launcher: diff --git a/Doc/whatsnew/3.7.rst b/Doc/whatsnew/3.7.rst index 25b1e1e33e3..7590af35e28 100644 --- a/Doc/whatsnew/3.7.rst +++ b/Doc/whatsnew/3.7.rst @@ -232,7 +232,8 @@ PEP 540: Forced UTF-8 Runtime Mode ----------------------------------- The new :option:`-X` ``utf8`` command line option and :envvar:`PYTHONUTF8` -environment variable can be used to enable the CPython *UTF-8 mode*. +environment variable can be used to enable the :ref:`Python UTF-8 Mode +`. When in UTF-8 mode, CPython ignores the locale settings, and uses the UTF-8 encoding by default. The error handlers for :data:`sys.stdin` and diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index dd5ca6121ca..3a38f77073d 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -127,273 +127,84 @@ PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ +/* This structure is best documented in the Doc/c-api/init_config.rst file. */ typedef struct { int _config_init; /* _PyConfigInitEnum value */ - int isolated; /* Isolated mode? see PyPreConfig.isolated */ - int use_environment; /* Use environment variables? see PyPreConfig.use_environment */ - int dev_mode; /* Python Development Mode? See PyPreConfig.dev_mode */ - - /* Install signal handlers? Yes by default. */ + int isolated; + int use_environment; + int dev_mode; int install_signal_handlers; - - int use_hash_seed; /* PYTHONHASHSEED=x */ + int use_hash_seed; unsigned long hash_seed; - - /* Enable faulthandler? - Set to 1 by -X faulthandler and PYTHONFAULTHANDLER. -1 means unset. */ int faulthandler; - - /* Enable tracemalloc? - Set by -X tracemalloc=N and PYTHONTRACEMALLOC. -1 means unset */ int tracemalloc; - - int import_time; /* PYTHONPROFILEIMPORTTIME, -X importtime */ - int show_ref_count; /* -X showrefcount */ - int dump_refs; /* PYTHONDUMPREFS */ - int malloc_stats; /* PYTHONMALLOCSTATS */ - - /* Python filesystem encoding and error handler: - sys.getfilesystemencoding() and sys.getfilesystemencodeerrors(). - - The Doc/c-api/init_config.rst documentation explains how Python selects - the filesystem encoding and error handler. - - _PyUnicode_InitEncodings() updates the encoding name to the Python codec - name. For example, "ANSI_X3.4-1968" is replaced with "ascii". It also - sets Py_FileSystemDefaultEncoding to filesystem_encoding and - sets Py_FileSystemDefaultEncodeErrors to filesystem_errors. */ + int import_time; + int show_ref_count; + int dump_refs; + int malloc_stats; wchar_t *filesystem_encoding; wchar_t *filesystem_errors; - - wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */ - int parse_argv; /* Parse argv command line arguments? */ - - /* Command line arguments (sys.argv). - - Set parse_argv to 1 to parse argv as Python command line arguments - and then strip Python arguments from argv. - - If argv is empty, an empty string is added to ensure that sys.argv - always exists and is never empty. */ + wchar_t *pycache_prefix; + int parse_argv; PyWideStringList argv; - - /* Program name: - - - If Py_SetProgramName() was called, use its value. - - On macOS, use PYTHONEXECUTABLE environment variable if set. - - If WITH_NEXT_FRAMEWORK macro is defined, use __PYVENV_LAUNCHER__ - environment variable is set. - - Use argv[0] if available and non-empty. - - Use "python" on Windows, or "python3 on other platforms. */ wchar_t *program_name; - - PyWideStringList xoptions; /* Command line -X options */ - - /* Warnings options: lowest to highest priority. warnings.filters - is built in the reverse order (highest to lowest priority). */ + PyWideStringList xoptions; PyWideStringList warnoptions; - - /* If equal to zero, disable the import of the module site and the - site-dependent manipulations of sys.path that it entails. Also disable - these manipulations if site is explicitly imported later (call - site.main() if you want them to be triggered). - - Set to 0 by the -S command line option. If set to -1 (default), it is - set to !Py_NoSiteFlag. */ int site_import; - - /* Bytes warnings: - - * If equal to 1, issue a warning when comparing bytes or bytearray with - str or bytes with int. - * If equal or greater to 2, issue an error. - - Incremented by the -b command line option. If set to -1 (default), inherit - Py_BytesWarningFlag value. */ int bytes_warning; - - /* If greater than 0, enable inspect: when a script is passed as first - argument or the -c option is used, enter interactive mode after - executing the script or the command, even when sys.stdin does not appear - to be a terminal. - - Incremented by the -i command line option. Set to 1 if the PYTHONINSPECT - environment variable is non-empty. If set to -1 (default), inherit - Py_InspectFlag value. */ int inspect; - - /* If greater than 0: enable the interactive mode (REPL). - - Incremented by the -i command line option. If set to -1 (default), - inherit Py_InteractiveFlag value. */ int interactive; - - /* Optimization level. - - Incremented by the -O command line option. Set by the PYTHONOPTIMIZE - environment variable. If set to -1 (default), inherit Py_OptimizeFlag - value. */ int optimization_level; - - /* If greater than 0, enable the debug mode: turn on parser debugging - output (for expert only, depending on compilation options). - - Incremented by the -d command line option. Set by the PYTHONDEBUG - environment variable. If set to -1 (default), inherit Py_DebugFlag - value. */ int parser_debug; - - /* If equal to 0, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set to 0 by the -B command line option and the PYTHONDONTWRITEBYTECODE - environment variable. If set to -1 (default), it is set to - !Py_DontWriteBytecodeFlag. */ int write_bytecode; - - /* If greater than 0, enable the verbose mode: print a message each time a - module is initialized, showing the place (filename or built-in module) - from which it is loaded. - - If greater or equal to 2, print a message for each file that is checked - for when searching for a module. Also provides information on module - cleanup at exit. - - Incremented by the -v option. Set by the PYTHONVERBOSE environment - variable. If set to -1 (default), inherit Py_VerboseFlag value. */ int verbose; - - /* If greater than 0, enable the quiet mode: Don't display the copyright - and version messages even in interactive mode. - - Incremented by the -q option. If set to -1 (default), inherit - Py_QuietFlag value. */ int quiet; - - /* If greater than 0, don't add the user site-packages directory to - sys.path. - - Set to 0 by the -s and -I command line options , and the PYTHONNOUSERSITE - environment variable. If set to -1 (default), it is set to - !Py_NoUserSiteDirectory. */ int user_site_directory; - - /* If non-zero, configure C standard steams (stdio, stdout, - stderr): - - - Set O_BINARY mode on Windows. - - If buffered_stdio is equal to zero, make streams unbuffered. - Otherwise, enable streams buffering if interactive is non-zero. */ int configure_c_stdio; - - /* If equal to 0, enable unbuffered mode: force the stdout and stderr - streams to be unbuffered. - - Set to 0 by the -u option. Set by the PYTHONUNBUFFERED environment - variable. - If set to -1 (default), it is set to !Py_UnbufferedStdioFlag. */ int buffered_stdio; - - /* Encoding of sys.stdin, sys.stdout and sys.stderr. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_errors' attribute. */ wchar_t *stdio_encoding; - - /* Error handler of sys.stdin and sys.stdout. - Value set from PYTHONIOENCODING environment variable and - Py_SetStandardStreamEncoding() function. - See also 'stdio_encoding' attribute. */ wchar_t *stdio_errors; - #ifdef MS_WINDOWS - /* If greater than zero, use io.FileIO instead of WindowsConsoleIO for sys - standard streams. - - Set to 1 if the PYTHONLEGACYWINDOWSSTDIO environment variable is set to - a non-empty string. If set to -1 (default), inherit - Py_LegacyWindowsStdioFlag value. - - See PEP 528 for more details. */ int legacy_windows_stdio; #endif - - /* Value of the --check-hash-based-pycs command line option: - - - "default" means the 'check_source' flag in hash-based pycs - determines invalidation - - "always" causes the interpreter to hash the source file for - invalidation regardless of value of 'check_source' bit - - "never" causes the interpreter to always assume hash-based pycs are - valid - - The default value is "default". - - See PEP 552 "Deterministic pycs" for more details. */ wchar_t *check_hash_pycs_mode; + PyWideStringList orig_argv; /* --- Path configuration inputs ------------ */ - - /* If greater than 0, suppress _PyPathConfig_Calculate() warnings on Unix. - The parameter has no effect on Windows. - - If set to -1 (default), inherit !Py_FrozenFlag value. */ int pathconfig_warnings; - - wchar_t *pythonpath_env; /* PYTHONPATH environment variable */ - wchar_t *home; /* PYTHONHOME environment variable, - see also Py_SetPythonHome(). */ + wchar_t *pythonpath_env; + wchar_t *home; /* --- Path configuration outputs ----------- */ - - int module_search_paths_set; /* If non-zero, use module_search_paths */ - PyWideStringList module_search_paths; /* sys.path paths. Computed if - module_search_paths_set is equal - to zero. */ - - wchar_t *executable; /* sys.executable */ - wchar_t *base_executable; /* sys._base_executable */ - wchar_t *prefix; /* sys.prefix */ - wchar_t *base_prefix; /* sys.base_prefix */ - wchar_t *exec_prefix; /* sys.exec_prefix */ - wchar_t *base_exec_prefix; /* sys.base_exec_prefix */ - wchar_t *platlibdir; /* sys.platlibdir */ + int module_search_paths_set; + PyWideStringList module_search_paths; + wchar_t *executable; + wchar_t *base_executable; + wchar_t *prefix; + wchar_t *base_prefix; + wchar_t *exec_prefix; + wchar_t *base_exec_prefix; + wchar_t *platlibdir; /* --- Parameter only used by Py_Main() ---------- */ - - /* Skip the first line of the source ('run_filename' parameter), allowing use of non-Unix forms of - "#!cmd". This is intended for a DOS specific hack only. - - Set by the -x command line option. */ int skip_source_first_line; - - wchar_t *run_command; /* -c command line argument */ - wchar_t *run_module; /* -m command line argument */ - wchar_t *run_filename; /* Trailing command line argument without -c or -m */ + wchar_t *run_command; + wchar_t *run_module; + wchar_t *run_filename; /* --- Private fields ---------------------------- */ - /* Install importlib? If set to 0, importlib is not initialized at all. - Needed by freeze_importlib. */ + // Install importlib? If equals to 0, importlib is not initialized at all. + // Needed by freeze_importlib. int _install_importlib; - /* If equal to 0, stop Python initialization before the "main" phase */ + // If equal to 0, stop Python initialization before the "main" phase. int _init_main; - /* If non-zero, disallow threads, subprocesses, and fork. - Default: 0. */ + // If non-zero, disallow threads, subprocesses, and fork. + // Default: 0. int _isolated_interpreter; - - /* The list of the original command line arguments passed to the Python - executable. - - If 'orig_argv' list is empty and 'argv' is not a list only containing an - empty string, PyConfig_Read() copies 'argv' into 'orig_argv' before - modifying 'argv' (if 'parse_argv is non-zero). - - _PyConfig_Write() initializes Py_GetArgcArgv() to this list. */ - PyWideStringList orig_argv; } PyConfig; PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config); diff --git a/Python/initconfig.c b/Python/initconfig.c index d0ff888c7f7..7bb28ed01f1 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -543,6 +543,7 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) } +// _PyConfig_Write() calls _Py_SetArgcArgv() with PyConfig.orig_argv. void Py_GetArgcArgv(int *argc, wchar_t ***argv) { diff --git a/Python/preconfig.c b/Python/preconfig.c index 149afcd99ab..b8b0c3a0775 100644 --- a/Python/preconfig.c +++ b/Python/preconfig.c @@ -19,11 +19,6 @@ preconfig_copy(PyPreConfig *config, const PyPreConfig *config2); /* --- File system encoding/errors -------------------------------- */ -/* The filesystem encoding is chosen by config_init_fs_encoding(), - see also initfsencoding(). - - Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors - are encoded to UTF-8. */ const char *Py_FileSystemDefaultEncoding = NULL; int Py_HasFileSystemDefaultEncoding = 0; const char *Py_FileSystemDefaultEncodeErrors = NULL; @@ -44,7 +39,10 @@ _Py_ClearFileSystemEncoding(void) /* Set Py_FileSystemDefaultEncoding and Py_FileSystemDefaultEncodeErrors - global configuration variables. */ + global configuration variables to PyConfig.filesystem_encoding and + PyConfig.filesystem_errors (encoded to UTF-8). + + Function called by _PyUnicode_InitEncodings(). */ int _Py_SetFileSystemEncoding(const char *encoding, const char *errors) { From aca67da4fe68d5420401ac1782203d302875eb27 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 2 Nov 2020 11:48:56 -0500 Subject: [PATCH 04/61] Revert "bpo-37193: remove thread objects which finished process its request (GH-13893)" (GH-23107) This reverts commit c41559021213cfc9dc62a83fc63306b3bdc3e64b. --- Lib/socketserver.py | 73 ++++--------------- Lib/test/test_socketserver.py | 24 ------ .../2020-06-12-21-23-20.bpo-37193.wJximU.rst | 2 - 3 files changed, 13 insertions(+), 86 deletions(-) delete mode 100644 Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst diff --git a/Lib/socketserver.py b/Lib/socketserver.py index 6859b69682e..57c1ae6e9e8 100644 --- a/Lib/socketserver.py +++ b/Lib/socketserver.py @@ -128,7 +128,6 @@ import selectors import os import sys import threading -import contextlib from io import BufferedIOBase from time import monotonic as time @@ -629,55 +628,6 @@ if hasattr(os, "fork"): self.collect_children(blocking=self.block_on_close) -class _Threads(list): - """ - Joinable list of all non-daemon threads. - """ - def __init__(self): - self._lock = threading.Lock() - - def append(self, thread): - if thread.daemon: - return - with self._lock: - super().append(thread) - - def remove(self, thread): - with self._lock: - # should not happen, but safe to ignore - with contextlib.suppress(ValueError): - super().remove(thread) - - def remove_current(self): - """Remove a current non-daemon thread.""" - thread = threading.current_thread() - if not thread.daemon: - self.remove(thread) - - def pop_all(self): - with self._lock: - self[:], result = [], self[:] - return result - - def join(self): - for thread in self.pop_all(): - thread.join() - - -class _NoThreads: - """ - Degenerate version of _Threads. - """ - def append(self, thread): - pass - - def join(self): - pass - - def remove_current(self): - pass - - class ThreadingMixIn: """Mix-in class to handle each request in a new thread.""" @@ -686,9 +636,9 @@ class ThreadingMixIn: daemon_threads = False # If true, server_close() waits until all non-daemonic threads terminate. block_on_close = True - # Threads object + # For non-daemonic threads, list of threading.Threading objects # used by server_close() to wait for all threads completion. - _threads = _NoThreads() + _threads = None def process_request_thread(self, request, client_address): """Same as in BaseServer but as a thread. @@ -701,24 +651,27 @@ class ThreadingMixIn: except Exception: self.handle_error(request, client_address) finally: - try: - self.shutdown_request(request) - finally: - self._threads.remove_current() + self.shutdown_request(request) def process_request(self, request, client_address): """Start a new thread to process the request.""" - if self.block_on_close: - vars(self).setdefault('_threads', _Threads()) t = threading.Thread(target = self.process_request_thread, args = (request, client_address)) t.daemon = self.daemon_threads - self._threads.append(t) + if not t.daemon and self.block_on_close: + if self._threads is None: + self._threads = [] + self._threads.append(t) t.start() def server_close(self): super().server_close() - self._threads.join() + if self.block_on_close: + threads = self._threads + self._threads = None + if threads: + for thread in threads: + thread.join() if hasattr(os, "fork"): diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 1944795f058..7cdd115a951 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -277,13 +277,6 @@ class SocketServerTest(unittest.TestCase): t.join() s.server_close() - def test_close_immediately(self): - class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - pass - - server = MyServer((HOST, 0), lambda: None) - server.server_close() - def test_tcpserver_bind_leak(self): # Issue #22435: the server socket wouldn't be closed if bind()/listen() # failed. @@ -498,23 +491,6 @@ class MiscTestCase(unittest.TestCase): self.assertEqual(server.shutdown_called, 1) server.server_close() - def test_threads_reaped(self): - """ - In #37193, users reported a memory leak - due to the saving of every request thread. Ensure that the - threads are cleaned up after the requests complete. - """ - class MyServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - pass - - server = MyServer((HOST, 0), socketserver.StreamRequestHandler) - for n in range(10): - with socket.create_connection(server.server_address): - server.handle_request() - [thread.join() for thread in server._threads] - self.assertEqual(len(server._threads), 0) - server.server_close() - if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst b/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst deleted file mode 100644 index fbf56d3194c..00000000000 --- a/Misc/NEWS.d/next/Library/2020-06-12-21-23-20.bpo-37193.wJximU.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed memory leak in ``socketserver.ThreadingMixIn`` introduced in Python -3.7. From 6fdfcec5b11f44f27aae3d53ddeb004150ae1f61 Mon Sep 17 00:00:00 2001 From: Irit Katriel Date: Mon, 2 Nov 2020 19:25:29 +0000 Subject: [PATCH 05/61] =?UTF-8?q?bpo-41943:=20Fix=20bug=20where=20assertLo?= =?UTF-8?q?gs=20doesn't=20correctly=20filter=20messages=E2=80=A6=20(GH-225?= =?UTF-8?q?65)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit … by level @vsajip , @pitrou Automerge-Triggered-By: GH:vsajip --- Lib/unittest/_log.py | 1 + Lib/unittest/test/test_case.py | 12 ++++++++++++ .../Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst | 1 + 3 files changed, 14 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst diff --git a/Lib/unittest/_log.py b/Lib/unittest/_log.py index 961c448a7fb..94868e5bb95 100644 --- a/Lib/unittest/_log.py +++ b/Lib/unittest/_log.py @@ -47,6 +47,7 @@ class _AssertLogsContext(_BaseTestCaseContext): logger = self.logger = logging.getLogger(self.logger_name) formatter = logging.Formatter(self.LOGGING_FORMAT) handler = _CapturingHandler() + handler.setLevel(self.level) handler.setFormatter(formatter) self.watcher = handler.watcher self.old_handlers = logger.handlers[:] diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py index 0e416967a30..b8aca92a8eb 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/unittest/test/test_case.py @@ -1673,6 +1673,18 @@ test case with self.assertLogs(level='WARNING'): log_foo.info("1") + def testAssertLogsFailureLevelTooHigh_FilterInRootLogger(self): + # Failure due to level too high - message propagated to root + with self.assertNoStderr(): + oldLevel = log_foo.level + log_foo.setLevel(logging.INFO) + try: + with self.assertRaises(self.failureException): + with self.assertLogs(level='WARNING'): + log_foo.info("1") + finally: + log_foo.setLevel(oldLevel) + def testAssertLogsFailureMismatchingLogger(self): # Failure due to mismatching logger (and the logged message is # passed through) diff --git a/Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst b/Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst new file mode 100644 index 00000000000..3a7874d25ae --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst @@ -0,0 +1 @@ +Fix bug where TestCase.assertLogs doesn't correctly filter messages by level. \ No newline at end of file From 34637a0ce21e7261b952fbd9d006474cc29b681f Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Mon, 2 Nov 2020 23:01:40 +0200 Subject: [PATCH 06/61] bpo-42103: Improve validation of Plist files. (GH-22882) * Prevent some possible DoS attacks via providing invalid Plist files with extremely large number of objects or collection sizes. * Raise InvalidFileException for too large bytes and string size instead of returning garbage. * Raise InvalidFileException instead of ValueError for specific invalid datetime (NaN). * Raise InvalidFileException instead of TypeError for non-hashable dict keys. * Add more tests for invalid Plist files. --- Lib/plistlib.py | 26 +- Lib/test/test_plistlib.py | 389 +++++++++++++++--- .../2020-10-23-19-20-14.bpo-42103.C5obK2.rst | 3 + .../2020-10-23-19-19-30.bpo-42103.cILT66.rst | 2 + 4 files changed, 363 insertions(+), 57 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst create mode 100644 Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst diff --git a/Lib/plistlib.py b/Lib/plistlib.py index a7403510a32..83b214e9dc4 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -477,7 +477,7 @@ class _BinaryPlistParser: return self._read_object(top_object) except (OSError, IndexError, struct.error, OverflowError, - UnicodeDecodeError): + ValueError): raise InvalidFileException() def _get_size(self, tokenL): @@ -493,7 +493,7 @@ class _BinaryPlistParser: def _read_ints(self, n, size): data = self._fp.read(size * n) if size in _BINARY_FORMAT: - return struct.unpack('>' + _BINARY_FORMAT[size] * n, data) + return struct.unpack(f'>{n}{_BINARY_FORMAT[size]}', data) else: if not size or len(data) != size * n: raise InvalidFileException() @@ -553,14 +553,22 @@ class _BinaryPlistParser: elif tokenH == 0x40: # data s = self._get_size(tokenL) result = self._fp.read(s) + if len(result) != s: + raise InvalidFileException() elif tokenH == 0x50: # ascii string s = self._get_size(tokenL) - result = self._fp.read(s).decode('ascii') + data = self._fp.read(s) + if len(data) != s: + raise InvalidFileException() + result = data.decode('ascii') elif tokenH == 0x60: # unicode string - s = self._get_size(tokenL) - result = self._fp.read(s * 2).decode('utf-16be') + s = self._get_size(tokenL) * 2 + data = self._fp.read(s) + if len(data) != s: + raise InvalidFileException() + result = data.decode('utf-16be') elif tokenH == 0x80: # UID # used by Key-Archiver plist files @@ -585,9 +593,11 @@ class _BinaryPlistParser: obj_refs = self._read_refs(s) result = self._dict_type() self._objects[ref] = result - for k, o in zip(key_refs, obj_refs): - result[self._read_object(k)] = self._read_object(o) - + try: + for k, o in zip(key_refs, obj_refs): + result[self._read_object(k)] = self._read_object(o) + except TypeError: + raise InvalidFileException() else: raise InvalidFileException() diff --git a/Lib/test/test_plistlib.py b/Lib/test/test_plistlib.py index c9dce0047b7..ef96c6ceda2 100644 --- a/Lib/test/test_plistlib.py +++ b/Lib/test/test_plistlib.py @@ -2,6 +2,7 @@ import copy import operator import pickle +import struct import unittest import plistlib import os @@ -119,6 +120,285 @@ XML_PLIST_WITH_ENTITY=b'''\ ''' +INVALID_BINARY_PLISTS = [ + ('too short data', + b'' + ), + ('too large offset_table_offset and offset_size = 1', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x2a' + ), + ('too large offset_table_offset and nonstandard offset_size', + b'\x00\x00\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x03\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x2c' + ), + ('integer overflow in offset_table_offset', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\xff\xff\xff\xff\xff\xff\xff\xff' + ), + ('too large top_object', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('integer overflow in top_object', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\xff\xff\xff\xff\xff\xff\xff\xff' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('too large num_objects and offset_size = 1', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\xff' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('too large num_objects and nonstandard offset_size', + b'\x00\x00\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x03\x01' + b'\x00\x00\x00\x00\x00\x00\x00\xff' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('extremally large num_objects (32 bit)', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x7f\xff\xff\xff' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('extremally large num_objects (64 bit)', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\xff\xff\xff\xff\xff' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('integer overflow in num_objects', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\xff\xff\xff\xff\xff\xff\xff\xff' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('offset_size = 0', + b'\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('ref_size = 0', + b'\xa1\x01\x00\x08\x0a' + b'\x00\x00\x00\x00\x00\x00\x01\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0b' + ), + ('too large offset', + b'\x00\x2a' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('integer overflow in offset', + b'\x00\xff\xff\xff\xff\xff\xff\xff\xff' + b'\x00\x00\x00\x00\x00\x00\x08\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x09' + ), + ('too large array size', + b'\xaf\x00\x01\xff\x00\x08\x0c' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0d' + ), + ('extremally large array size (32-bit)', + b'\xaf\x02\x7f\xff\xff\xff\x01\x00\x08\x0f' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x10' + ), + ('extremally large array size (64-bit)', + b'\xaf\x03\x00\x00\x00\xff\xff\xff\xff\xff\x01\x00\x08\x13' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x14' + ), + ('integer overflow in array size', + b'\xaf\x03\xff\xff\xff\xff\xff\xff\xff\xff\x01\x00\x08\x13' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x14' + ), + ('too large reference index', + b'\xa1\x02\x00\x08\x0a' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0b' + ), + ('integer overflow in reference index', + b'\xa1\xff\xff\xff\xff\xff\xff\xff\xff\x00\x08\x11' + b'\x00\x00\x00\x00\x00\x00\x01\x08' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x12' + ), + ('too large bytes size', + b'\x4f\x00\x23\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0c' + ), + ('extremally large bytes size (32-bit)', + b'\x4f\x02\x7f\xff\xff\xff\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0f' + ), + ('extremally large bytes size (64-bit)', + b'\x4f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x13' + ), + ('integer overflow in bytes size', + b'\x4f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x13' + ), + ('too large ASCII size', + b'\x5f\x00\x23\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0c' + ), + ('extremally large ASCII size (32-bit)', + b'\x5f\x02\x7f\xff\xff\xff\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0f' + ), + ('extremally large ASCII size (64-bit)', + b'\x5f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x13' + ), + ('integer overflow in ASCII size', + b'\x5f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x41\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x13' + ), + ('invalid ASCII', + b'\x51\xff\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0a' + ), + ('too large UTF-16 size', + b'\x6f\x00\x13\x20\xac\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0e' + ), + ('extremally large UTF-16 size (32-bit)', + b'\x6f\x02\x4f\xff\xff\xff\x20\xac\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x11' + ), + ('extremally large UTF-16 size (64-bit)', + b'\x6f\x03\x00\x00\x00\xff\xff\xff\xff\xff\x20\xac\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x15' + ), + ('integer overflow in UTF-16 size', + b'\x6f\x03\xff\xff\xff\xff\xff\xff\xff\xff\x20\xac\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x15' + ), + ('invalid UTF-16', + b'\x61\xd8\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0b' + ), + ('non-hashable key', + b'\xd1\x01\x01\xa0\x08\x0b' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x02' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x0c' + ), + ('too large datetime (datetime overflow)', + b'\x33\x42\x50\x00\x00\x00\x00\x00\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x11' + ), + ('too large datetime (timedelta overflow)', + b'\x33\x42\xe0\x00\x00\x00\x00\x00\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x11' + ), + ('invalid datetime (Infinity)', + b'\x33\x7f\xf0\x00\x00\x00\x00\x00\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x11' + ), + ('invalid datetime (NaN)', + b'\x33\x7f\xf8\x00\x00\x00\x00\x00\x00\x08' + b'\x00\x00\x00\x00\x00\x00\x01\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x01' + b'\x00\x00\x00\x00\x00\x00\x00\x00' + b'\x00\x00\x00\x00\x00\x00\x00\x11' + ), +] + class TestPlistlib(unittest.TestCase): @@ -558,6 +838,21 @@ class TestPlistlib(unittest.TestCase): class TestBinaryPlistlib(unittest.TestCase): + @staticmethod + def decode(*objects, offset_size=1, ref_size=1): + data = [b'bplist00'] + offset = 8 + offsets = [] + for x in objects: + offsets.append(offset.to_bytes(offset_size, 'big')) + data.append(x) + offset += len(x) + tail = struct.pack('>6xBBQQQ', offset_size, ref_size, + len(objects), 0, offset) + data.extend(offsets) + data.append(tail) + return plistlib.loads(b''.join(data), fmt=plistlib.FMT_BINARY) + def test_nonstandard_refs_size(self): # Issue #21538: Refs and offsets are 24-bit integers data = (b'bplist00' @@ -572,7 +867,7 @@ class TestBinaryPlistlib(unittest.TestCase): def test_dump_duplicates(self): # Test effectiveness of saving duplicated objects - for x in (None, False, True, 12345, 123.45, 'abcde', b'abcde', + for x in (None, False, True, 12345, 123.45, 'abcde', 'абвгд', b'abcde', datetime.datetime(2004, 10, 26, 10, 33, 33), bytearray(b'abcde'), [12, 345], (12, 345), {'12': 345}): with self.subTest(x=x): @@ -609,6 +904,20 @@ class TestBinaryPlistlib(unittest.TestCase): b = plistlib.loads(plistlib.dumps(a, fmt=plistlib.FMT_BINARY)) self.assertIs(b['x'], b) + def test_deep_nesting(self): + for N in [300, 100000]: + chunks = [b'\xa1' + (i + 1).to_bytes(4, 'big') for i in range(N)] + try: + result = self.decode(*chunks, b'\x54seed', offset_size=4, ref_size=4) + except RecursionError: + pass + else: + for i in range(N): + self.assertIsInstance(result, list) + self.assertEqual(len(result), 1) + result = result[0] + self.assertEqual(result, 'seed') + def test_large_timestamp(self): # Issue #26709: 32-bit timestamp out of range for ts in -2**31-1, 2**31: @@ -618,55 +927,37 @@ class TestBinaryPlistlib(unittest.TestCase): data = plistlib.dumps(d, fmt=plistlib.FMT_BINARY) self.assertEqual(plistlib.loads(data), d) + def test_load_singletons(self): + self.assertIs(self.decode(b'\x00'), None) + self.assertIs(self.decode(b'\x08'), False) + self.assertIs(self.decode(b'\x09'), True) + self.assertEqual(self.decode(b'\x0f'), b'') + + def test_load_int(self): + self.assertEqual(self.decode(b'\x10\x00'), 0) + self.assertEqual(self.decode(b'\x10\xfe'), 0xfe) + self.assertEqual(self.decode(b'\x11\xfe\xdc'), 0xfedc) + self.assertEqual(self.decode(b'\x12\xfe\xdc\xba\x98'), 0xfedcba98) + self.assertEqual(self.decode(b'\x13\x01\x23\x45\x67\x89\xab\xcd\xef'), + 0x0123456789abcdef) + self.assertEqual(self.decode(b'\x13\xfe\xdc\xba\x98\x76\x54\x32\x10'), + -0x123456789abcdf0) + + def test_unsupported(self): + unsupported = [*range(1, 8), *range(10, 15), + 0x20, 0x21, *range(0x24, 0x33), *range(0x34, 0x40)] + for i in [0x70, 0x90, 0xb0, 0xc0, 0xe0, 0xf0]: + unsupported.extend(i + j for j in range(16)) + for token in unsupported: + with self.subTest(f'token {token:02x}'): + with self.assertRaises(plistlib.InvalidFileException): + self.decode(bytes([token]) + b'\x00'*16) + def test_invalid_binary(self): - for data in [ - # too short data - b'', - # too large offset_table_offset and nonstandard offset_size - b'\x00\x08' - b'\x00\x00\x00\x00\x00\x00\x03\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x2a', - # integer overflow in offset_table_offset - b'\x00\x08' - b'\x00\x00\x00\x00\x00\x00\x01\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\xff\xff\xff\xff\xff\xff\xff\xff', - # offset_size = 0 - b'\x00\x08' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x09', - # ref_size = 0 - b'\xa1\x01\x00\x08\x0a' - b'\x00\x00\x00\x00\x00\x00\x01\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x02' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x0b', - # integer overflow in offset - b'\x00\xff\xff\xff\xff\xff\xff\xff\xff' - b'\x00\x00\x00\x00\x00\x00\x08\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x09', - # invalid ASCII - b'\x51\xff\x08' - b'\x00\x00\x00\x00\x00\x00\x01\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x0a', - # invalid UTF-16 - b'\x61\xd8\x00\x08' - b'\x00\x00\x00\x00\x00\x00\x01\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x01' - b'\x00\x00\x00\x00\x00\x00\x00\x00' - b'\x00\x00\x00\x00\x00\x00\x00\x0b', - ]: - with self.assertRaises(plistlib.InvalidFileException): - plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) + for name, data in INVALID_BINARY_PLISTS: + with self.subTest(name): + with self.assertRaises(plistlib.InvalidFileException): + plistlib.loads(b'bplist00' + data, fmt=plistlib.FMT_BINARY) class TestKeyedArchive(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst new file mode 100644 index 00000000000..4eb694c16a0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst @@ -0,0 +1,3 @@ +:exc:`~plistlib.InvalidFileException` and :exc:`RecursionError` are now +the only errors caused by loading malformed binary Plist file (previously +ValueError and TypeError could be raised in some specific cases). diff --git a/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst new file mode 100644 index 00000000000..15d7b6549ed --- /dev/null +++ b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst @@ -0,0 +1,2 @@ +Prevented potential DoS attack via CPU and RAM exhaustion when processing +malformed Apple Property List files in binary format. From 5cf4782a2630629d0978bf4cf6b6340365f449b2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Nov 2020 22:03:28 +0100 Subject: [PATCH 07/61] bpo-41796: Make _ast module state per interpreter (GH-23024) The ast module internal state is now per interpreter. * Rename "astmodulestate" to "struct ast_state" * Add pycore_ast.h internal header: the ast_state structure is now declared in pycore_ast.h. * Add PyInterpreterState.ast (struct ast_state) * Remove get_ast_state() * Rename get_global_ast_state() to get_ast_state() * PyAST_obj2mod() now handles get_ast_state() failures --- Include/Python-ast.h | 2 +- Include/internal/pycore_ast.h | 234 +++++++++++++++++ Include/internal/pycore_interp.h | 11 +- Makefile.pre.in | 2 + .../2020-10-29-12-49-08.bpo-41796.tkGdHq.rst | 2 + PCbuild/regen.vcxproj | 12 +- Parser/asdl_c.py | 243 +++++++++++------ Python/Python-ast.c | 245 ++++++++++-------- 8 files changed, 549 insertions(+), 202 deletions(-) create mode 100644 Include/internal/pycore_ast.h create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst diff --git a/Include/Python-ast.h b/Include/Python-ast.h index e14bab566fb..fc9f65c97a2 100644 --- a/Include/Python-ast.h +++ b/Include/Python-ast.h @@ -1,4 +1,4 @@ -/* File automatically generated by Parser/asdl_c.py. */ +// File automatically generated by Parser/asdl_c.py. #ifndef Py_PYTHON_AST_H #define Py_PYTHON_AST_H diff --git a/Include/internal/pycore_ast.h b/Include/internal/pycore_ast.h new file mode 100644 index 00000000000..058fbc0fcdc --- /dev/null +++ b/Include/internal/pycore_ast.h @@ -0,0 +1,234 @@ +// File automatically generated by Parser/asdl_c.py. + +#ifndef Py_INTERNAL_AST_H +#define Py_INTERNAL_AST_H +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef Py_BUILD_CORE +# error "this header requires Py_BUILD_CORE define" +#endif + +struct ast_state { + int initialized; + PyObject *AST_type; + PyObject *Add_singleton; + PyObject *Add_type; + PyObject *And_singleton; + PyObject *And_type; + PyObject *AnnAssign_type; + PyObject *Assert_type; + PyObject *Assign_type; + PyObject *AsyncFor_type; + PyObject *AsyncFunctionDef_type; + PyObject *AsyncWith_type; + PyObject *Attribute_type; + PyObject *AugAssign_type; + PyObject *Await_type; + PyObject *BinOp_type; + PyObject *BitAnd_singleton; + PyObject *BitAnd_type; + PyObject *BitOr_singleton; + PyObject *BitOr_type; + PyObject *BitXor_singleton; + PyObject *BitXor_type; + PyObject *BoolOp_type; + PyObject *Break_type; + PyObject *Call_type; + PyObject *ClassDef_type; + PyObject *Compare_type; + PyObject *Constant_type; + PyObject *Continue_type; + PyObject *Del_singleton; + PyObject *Del_type; + PyObject *Delete_type; + PyObject *DictComp_type; + PyObject *Dict_type; + PyObject *Div_singleton; + PyObject *Div_type; + PyObject *Eq_singleton; + PyObject *Eq_type; + PyObject *ExceptHandler_type; + PyObject *Expr_type; + PyObject *Expression_type; + PyObject *FloorDiv_singleton; + PyObject *FloorDiv_type; + PyObject *For_type; + PyObject *FormattedValue_type; + PyObject *FunctionDef_type; + PyObject *FunctionType_type; + PyObject *GeneratorExp_type; + PyObject *Global_type; + PyObject *GtE_singleton; + PyObject *GtE_type; + PyObject *Gt_singleton; + PyObject *Gt_type; + PyObject *IfExp_type; + PyObject *If_type; + PyObject *ImportFrom_type; + PyObject *Import_type; + PyObject *In_singleton; + PyObject *In_type; + PyObject *Interactive_type; + PyObject *Invert_singleton; + PyObject *Invert_type; + PyObject *IsNot_singleton; + PyObject *IsNot_type; + PyObject *Is_singleton; + PyObject *Is_type; + PyObject *JoinedStr_type; + PyObject *LShift_singleton; + PyObject *LShift_type; + PyObject *Lambda_type; + PyObject *ListComp_type; + PyObject *List_type; + PyObject *Load_singleton; + PyObject *Load_type; + PyObject *LtE_singleton; + PyObject *LtE_type; + PyObject *Lt_singleton; + PyObject *Lt_type; + PyObject *MatMult_singleton; + PyObject *MatMult_type; + PyObject *Mod_singleton; + PyObject *Mod_type; + PyObject *Module_type; + PyObject *Mult_singleton; + PyObject *Mult_type; + PyObject *Name_type; + PyObject *NamedExpr_type; + PyObject *Nonlocal_type; + PyObject *NotEq_singleton; + PyObject *NotEq_type; + PyObject *NotIn_singleton; + PyObject *NotIn_type; + PyObject *Not_singleton; + PyObject *Not_type; + PyObject *Or_singleton; + PyObject *Or_type; + PyObject *Pass_type; + PyObject *Pow_singleton; + PyObject *Pow_type; + PyObject *RShift_singleton; + PyObject *RShift_type; + PyObject *Raise_type; + PyObject *Return_type; + PyObject *SetComp_type; + PyObject *Set_type; + PyObject *Slice_type; + PyObject *Starred_type; + PyObject *Store_singleton; + PyObject *Store_type; + PyObject *Sub_singleton; + PyObject *Sub_type; + PyObject *Subscript_type; + PyObject *Try_type; + PyObject *Tuple_type; + PyObject *TypeIgnore_type; + PyObject *UAdd_singleton; + PyObject *UAdd_type; + PyObject *USub_singleton; + PyObject *USub_type; + PyObject *UnaryOp_type; + PyObject *While_type; + PyObject *With_type; + PyObject *YieldFrom_type; + PyObject *Yield_type; + PyObject *__dict__; + PyObject *__doc__; + PyObject *__module__; + PyObject *_attributes; + PyObject *_fields; + PyObject *alias_type; + PyObject *annotation; + PyObject *arg; + PyObject *arg_type; + PyObject *args; + PyObject *argtypes; + PyObject *arguments_type; + PyObject *asname; + PyObject *ast; + PyObject *attr; + PyObject *bases; + PyObject *body; + PyObject *boolop_type; + PyObject *cause; + PyObject *cmpop_type; + PyObject *col_offset; + PyObject *comparators; + PyObject *comprehension_type; + PyObject *context_expr; + PyObject *conversion; + PyObject *ctx; + PyObject *decorator_list; + PyObject *defaults; + PyObject *elt; + PyObject *elts; + PyObject *end_col_offset; + PyObject *end_lineno; + PyObject *exc; + PyObject *excepthandler_type; + PyObject *expr_context_type; + PyObject *expr_type; + PyObject *finalbody; + PyObject *format_spec; + PyObject *func; + PyObject *generators; + PyObject *handlers; + PyObject *id; + PyObject *ifs; + PyObject *is_async; + PyObject *items; + PyObject *iter; + PyObject *key; + PyObject *keys; + PyObject *keyword_type; + PyObject *keywords; + PyObject *kind; + PyObject *kw_defaults; + PyObject *kwarg; + PyObject *kwonlyargs; + PyObject *left; + PyObject *level; + PyObject *lineno; + PyObject *lower; + PyObject *mod_type; + PyObject *module; + PyObject *msg; + PyObject *name; + PyObject *names; + PyObject *op; + PyObject *operand; + PyObject *operator_type; + PyObject *ops; + PyObject *optional_vars; + PyObject *orelse; + PyObject *posonlyargs; + PyObject *returns; + PyObject *right; + PyObject *simple; + PyObject *slice; + PyObject *step; + PyObject *stmt_type; + PyObject *tag; + PyObject *target; + PyObject *targets; + PyObject *test; + PyObject *type; + PyObject *type_comment; + PyObject *type_ignore_type; + PyObject *type_ignores; + PyObject *unaryop_type; + PyObject *upper; + PyObject *value; + PyObject *values; + PyObject *vararg; + PyObject *withitem_type; +}; + +#ifdef __cplusplus +} +#endif +#endif /* !Py_INTERNAL_AST_H */ + diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 69d2108da43..9923b6b03da 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -8,10 +8,11 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif -#include "pycore_atomic.h" /* _Py_atomic_address */ -#include "pycore_gil.h" /* struct _gil_runtime_state */ -#include "pycore_gc.h" /* struct _gc_runtime_state */ -#include "pycore_warnings.h" /* struct _warnings_runtime_state */ +#include "pycore_atomic.h" // _Py_atomic_address +#include "pycore_ast.h" // struct ast_state +#include "pycore_gil.h" // struct _gil_runtime_state +#include "pycore_gc.h" // struct _gc_runtime_state +#include "pycore_warnings.h" // struct _warnings_runtime_state struct _pending_calls { PyThread_type_lock lock; @@ -258,6 +259,8 @@ struct _is { struct _Py_async_gen_state async_gen; struct _Py_context_state context; struct _Py_exc_state exc_state; + + struct ast_state ast; }; /* Used by _PyImport_Cleanup() */ diff --git a/Makefile.pre.in b/Makefile.pre.in index 5b6c0b9b62d..a8912cd4184 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -840,9 +840,11 @@ regen-ast: $(PYTHON_FOR_REGEN) $(srcdir)/Parser/asdl_c.py \ $(srcdir)/Parser/Python.asdl \ -H $(srcdir)/Include/Python-ast.h.new \ + -I $(srcdir)/Include/internal/pycore_ast.h.new \ -C $(srcdir)/Python/Python-ast.c.new $(UPDATE_FILE) $(srcdir)/Include/Python-ast.h $(srcdir)/Include/Python-ast.h.new + $(UPDATE_FILE) $(srcdir)/Include/internal/pycore_ast.h $(srcdir)/Include/internal/pycore_ast.h.new $(UPDATE_FILE) $(srcdir)/Python/Python-ast.c $(srcdir)/Python/Python-ast.c.new .PHONY: regen-opcode diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst new file mode 100644 index 00000000000..b3ac08edd84 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst @@ -0,0 +1,2 @@ +The :mod:`ast` module internal state is now per interpreter. Patch by Victor +Stinner. diff --git a/PCbuild/regen.vcxproj b/PCbuild/regen.vcxproj index 90d6dc68d54..936f5fd2464 100644 --- a/PCbuild/regen.vcxproj +++ b/PCbuild/regen.vcxproj @@ -137,6 +137,8 @@ + + @@ -161,15 +163,18 @@ - - + + - + + + + @@ -204,6 +209,7 @@ + diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 481261cd853..9a833e841de 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -3,6 +3,7 @@ import os import sys +import textwrap from argparse import ArgumentParser from pathlib import Path @@ -11,7 +12,7 @@ import asdl TABSIZE = 4 MAX_COL = 80 -AUTOGEN_MESSAGE = "/* File automatically generated by {}. */\n\n" +AUTOGEN_MESSAGE = "// File automatically generated by {}.\n\n" def get_c_type(name): """Return a string for the C name of the type. @@ -414,7 +415,7 @@ class PickleVisitor(EmitVisitor): class Obj2ModPrototypeVisitor(PickleVisitor): def visitProduct(self, prod, name): - code = "static int obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena);" + code = "static int obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena);" self.emit(code % (name, get_c_type(name)), 0) visitSum = visitProduct @@ -424,7 +425,7 @@ class Obj2ModVisitor(PickleVisitor): def funcHeader(self, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("int isinstance;", 1) self.emit("", 0) @@ -506,7 +507,7 @@ class Obj2ModVisitor(PickleVisitor): def visitProduct(self, prod, name): ctype = get_c_type(name) self.emit("int", 0) - self.emit("obj2ast_%s(astmodulestate *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) + self.emit("obj2ast_%s(struct ast_state *state, PyObject* obj, %s* out, PyArena* arena)" % (name, ctype), 0) self.emit("{", 0) self.emit("PyObject* tmp = NULL;", 1) for f in prod.fields: @@ -640,7 +641,7 @@ class PyTypesDeclareVisitor(PickleVisitor): def visitProduct(self, prod, name): self.emit_type("%s_type" % name) - self.emit("static PyObject* ast2obj_%s(astmodulestate *state, void*);" % name, 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, void*);" % name, 0) if prod.attributes: for a in prod.attributes: self.emit_identifier(a.name) @@ -670,7 +671,7 @@ class PyTypesDeclareVisitor(PickleVisitor): ptype = get_c_type(name) for t in sum.types: self.emit_singleton("%s_singleton" % t.name) - self.emit("static PyObject* ast2obj_%s(astmodulestate *state, %s);" % (name, ptype), 0) + self.emit("static PyObject* ast2obj_%s(struct ast_state *state, %s);" % (name, ptype), 0) for t in sum.types: self.visitConstructor(t, name) @@ -725,7 +726,7 @@ ast_clear(AST_object *self) static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -801,7 +802,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -856,7 +857,7 @@ static PyType_Spec AST_type_spec = { }; static PyObject * -make_type(astmodulestate *state, const char *type, PyObject* base, +make_type(struct ast_state *state, const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) { PyObject *fnames, *result; @@ -882,7 +883,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base, } static int -add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) +add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); @@ -903,7 +904,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, /* Conversion AST -> Python */ -static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -921,7 +922,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (* return result; } -static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) { if (!o) o = Py_None; @@ -932,14 +933,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) { return PyLong_FromLong(b); } /* Conversion Python -> AST */ -static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -954,7 +955,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj return 0; } -static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -965,7 +966,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO return 0; } -static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); @@ -974,7 +975,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o return obj2ast_object(state, obj, out, arena); } -static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); @@ -983,7 +984,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, return obj2ast_object(state, obj, out, arena); } -static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) { int i; if (!PyLong_Check(obj)) { @@ -998,7 +999,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out return 0; } -static int add_ast_fields(astmodulestate *state) +static int add_ast_fields(struct ast_state *state) { PyObject *empty_tuple; empty_tuple = PyTuple_New(0); @@ -1014,7 +1015,7 @@ static int add_ast_fields(astmodulestate *state) """, 0, reflow=False) - self.emit("static int init_types(astmodulestate *state)",0) + self.emit("static int init_types(struct ast_state *state)",0) self.emit("{", 0) self.emit("if (state->initialized) return 1;", 1) self.emit("if (init_identifiers(state) < 0) return 0;", 1) @@ -1093,12 +1094,10 @@ class ASTModuleVisitor(PickleVisitor): self.emit("static int", 0) self.emit("astmodule_exec(PyObject *m)", 0) self.emit("{", 0) - self.emit('astmodulestate *state = get_ast_state(m);', 1) - self.emit("", 0) - - self.emit("if (!init_types(state)) {", 1) - self.emit("return -1;", 2) - self.emit("}", 1) + self.emit('struct ast_state *state = get_ast_state();', 1) + self.emit('if (state == NULL) {', 1) + self.emit('return -1;', 2) + self.emit('}', 1) self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) self.emit('return -1;', 2) self.emit('}', 1) @@ -1126,7 +1125,7 @@ static PyModuleDef_Slot astmodule_slots[] = { static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, .m_name = "_ast", - // The _ast module uses a global state (global_ast_state). + // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, .m_slots = astmodule_slots, }; @@ -1169,7 +1168,7 @@ class ObjVisitor(PickleVisitor): def func_begin(self, name): ctype = get_c_type(name) self.emit("PyObject*", 0) - self.emit("ast2obj_%s(astmodulestate *state, void* _o)" % (name), 0) + self.emit("ast2obj_%s(struct ast_state *state, void* _o)" % (name), 0) self.emit("{", 0) self.emit("%s o = (%s)_o;" % (ctype, ctype), 1) self.emit("PyObject *result = NULL, *value = NULL;", 1) @@ -1206,7 +1205,7 @@ class ObjVisitor(PickleVisitor): self.func_end() def simpleSum(self, sum, name): - self.emit("PyObject* ast2obj_%s(astmodulestate *state, %s_ty o)" % (name, name), 0) + self.emit("PyObject* ast2obj_%s(struct ast_state *state, %s_ty o)" % (name, name), 0) self.emit("{", 0) self.emit("switch(o) {", 1) for t in sum.types: @@ -1280,7 +1279,7 @@ class PartingShots(StaticVisitor): CODE = """ PyObject* PyAST_mod2obj(mod_ty t) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -1297,7 +1296,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) return NULL; } - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -1323,7 +1326,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -1341,7 +1344,35 @@ class ChainOfVisitors: v.emit("", 0) -def generate_module_def(f, mod): +def generate_ast_state(module_state, f): + f.write('struct ast_state {\n') + f.write(' int initialized;\n') + for s in module_state: + f.write(' PyObject *' + s + ';\n') + f.write('};') + + +def generate_ast_fini(module_state, f): + f.write(""" +void _PyAST_Fini(PyThreadState *tstate) +{ +#ifdef Py_BUILD_CORE + struct ast_state *state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + +""") + for s in module_state: + f.write(" Py_CLEAR(state->" + s + ');\n') + f.write(""" + state->initialized = 0; +} + +""") + + +def generate_module_def(mod, f, internal_h): # Gather all the data needed for ModuleSpec visitor_list = set() with open(os.devnull, "w") as devnull: @@ -1371,50 +1402,64 @@ def generate_module_def(f, mod): module_state.add(tp) state_strings = sorted(state_strings) module_state = sorted(module_state) - f.write('typedef struct {\n') - f.write(' int initialized;\n') - for s in module_state: - f.write(' PyObject *' + s + ';\n') - f.write('} astmodulestate;\n\n') + + generate_ast_state(module_state, internal_h) + + print(textwrap.dedent(f""" + #ifdef Py_BUILD_CORE + # include "pycore_ast.h" // struct ast_state + # include "pycore_interp.h" // _PyInterpreterState.ast + # include "pycore_pystate.h" // _PyInterpreterState_GET() + #else + """).strip(), file=f) + + generate_ast_state(module_state, f) + + print(textwrap.dedent(f""" + #endif // Py_BUILD_CORE + """).rstrip(), file=f) + f.write(""" // Forward declaration -static int init_types(astmodulestate *state); +static int init_types(struct ast_state *state); -// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. -static astmodulestate global_ast_state = {0}; - -static astmodulestate* -get_global_ast_state(void) +#ifdef Py_BUILD_CORE +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = &global_ast_state; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct ast_state *state = &interp->ast; if (!init_types(state)) { return NULL; } return state; } +#else +static struct ast_state global_ast_state; -static astmodulestate* -get_ast_state(PyObject* Py_UNUSED(module)) +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = get_global_ast_state(); - // get_ast_state() must only be called after _ast module is imported, - // and astmodule_exec() calls init_types() - assert(state != NULL); + struct ast_state *state = &global_ast_state; + if (!init_types(state)) { + return NULL; + } return state; } - -void _PyAST_Fini(PyThreadState *tstate) -{ - astmodulestate* state = &global_ast_state; +#endif // Py_BUILD_CORE """) - for s in module_state: - f.write(" Py_CLEAR(state->" + s + ');\n') - f.write(""" - state->initialized = 0; -} + # f-string for {mod.name} + f.write(f""" +// Include {mod.name}-ast.h after pycore_interp.h to avoid conflicts +// with the Yield macro redefined by +#include "{mod.name}-ast.h" +#include "structmember.h" """) - f.write('static int init_identifiers(astmodulestate *state)\n') + + generate_ast_fini(module_state, f) + + f.write('static int init_identifiers(struct ast_state *state)\n') f.write('{\n') for identifier in state_strings: f.write(' if ((state->' + identifier) @@ -1423,7 +1468,7 @@ void _PyAST_Fini(PyThreadState *tstate) f.write(' return 1;\n') f.write('};\n\n') -def write_header(f, mod): +def write_header(mod, f): f.write('#ifndef Py_PYTHON_AST_H\n') f.write('#define Py_PYTHON_AST_H\n') f.write('#ifdef __cplusplus\n') @@ -1452,15 +1497,39 @@ def write_header(f, mod): f.write('#endif\n') f.write('#endif /* !Py_PYTHON_AST_H */\n') -def write_source(f, mod): - f.write('#include \n') - f.write('\n') - f.write('#include "Python.h"\n') - f.write('#include "%s-ast.h"\n' % mod.name) - f.write('#include "structmember.h" // PyMemberDef\n') - f.write('\n') - generate_module_def(f, mod) +def write_internal_h_header(mod, f): + print(textwrap.dedent(""" + #ifndef Py_INTERNAL_AST_H + #define Py_INTERNAL_AST_H + #ifdef __cplusplus + extern "C" { + #endif + + #ifndef Py_BUILD_CORE + # error "this header requires Py_BUILD_CORE define" + #endif + """).lstrip(), file=f) + + +def write_internal_h_footer(mod, f): + print(textwrap.dedent(""" + + #ifdef __cplusplus + } + #endif + #endif /* !Py_INTERNAL_AST_H */ + """), file=f) + + +def write_source(mod, f, internal_h_file): + print(textwrap.dedent(f""" + #include + + #include "Python.h" + """), file=f) + + generate_module_def(mod, f, internal_h_file) v = ChainOfVisitors( SequenceConstructorVisitor(f), @@ -1475,27 +1544,37 @@ def write_source(f, mod): ) v.visit(mod) -def main(input_file, c_file, h_file, dump_module=False): +def main(input_filename, c_filename, h_filename, internal_h_filename, dump_module=False): auto_gen_msg = AUTOGEN_MESSAGE.format("/".join(Path(__file__).parts[-2:])) - mod = asdl.parse(input_file) + mod = asdl.parse(input_filename) if dump_module: print('Parsed Module:') print(mod) if not asdl.check(mod): sys.exit(1) - for file, writer in (c_file, write_source), (h_file, write_header): - if file is not None: - with file.open("w") as f: - f.write(auto_gen_msg) - writer(f, mod) - print(file, "regenerated.") + + with c_filename.open("w") as c_file, \ + h_filename.open("w") as h_file, \ + internal_h_filename.open("w") as internal_h_file: + c_file.write(auto_gen_msg) + h_file.write(auto_gen_msg) + internal_h_file.write(auto_gen_msg) + + write_internal_h_header(mod, internal_h_file) + write_source(mod, c_file, internal_h_file) + write_header(mod, h_file) + write_internal_h_footer(mod, internal_h_file) + + print(f"{c_filename}, {h_filename}, {internal_h_filename} regenerated.") if __name__ == "__main__": parser = ArgumentParser() parser.add_argument("input_file", type=Path) - parser.add_argument("-C", "--c-file", type=Path, default=None) - parser.add_argument("-H", "--h-file", type=Path, default=None) + parser.add_argument("-C", "--c-file", type=Path, required=True) + parser.add_argument("-H", "--h-file", type=Path, required=True) + parser.add_argument("-I", "--internal-h-file", type=Path, required=True) parser.add_argument("-d", "--dump-module", action="store_true") - options = parser.parse_args() - main(**vars(options)) + args = parser.parse_args() + main(args.input_file, args.c_file, args.h_file, + args.internal_h_file, args.dump_module) diff --git a/Python/Python-ast.c b/Python/Python-ast.c index 13657a67275..f04addbe201 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -1,12 +1,16 @@ -/* File automatically generated by Parser/asdl_c.py. */ +// File automatically generated by Parser/asdl_c.py. + #include #include "Python.h" -#include "Python-ast.h" -#include "structmember.h" // PyMemberDef -typedef struct { +#ifdef Py_BUILD_CORE +# include "pycore_ast.h" // struct ast_state +# include "pycore_interp.h" // _PyInterpreterState.ast +# include "pycore_pystate.h" // _PyInterpreterState_GET() +#else +struct ast_state { int initialized; PyObject *AST_type; PyObject *Add_singleton; @@ -221,38 +225,50 @@ typedef struct { PyObject *values; PyObject *vararg; PyObject *withitem_type; -} astmodulestate; - +}; +#endif // Py_BUILD_CORE // Forward declaration -static int init_types(astmodulestate *state); +static int init_types(struct ast_state *state); -// bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. -static astmodulestate global_ast_state = {0}; - -static astmodulestate* -get_global_ast_state(void) +#ifdef Py_BUILD_CORE +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = &global_ast_state; + PyInterpreterState *interp = _PyInterpreterState_GET(); + struct ast_state *state = &interp->ast; if (!init_types(state)) { return NULL; } return state; } +#else +static struct ast_state global_ast_state; -static astmodulestate* -get_ast_state(PyObject* Py_UNUSED(module)) +static struct ast_state* +get_ast_state(void) { - astmodulestate* state = get_global_ast_state(); - // get_ast_state() must only be called after _ast module is imported, - // and astmodule_exec() calls init_types() - assert(state != NULL); + struct ast_state *state = &global_ast_state; + if (!init_types(state)) { + return NULL; + } return state; } +#endif // Py_BUILD_CORE + +// Include Python-ast.h after pycore_interp.h to avoid conflicts +// with the Yield macro redefined by +#include "Python-ast.h" +#include "structmember.h" void _PyAST_Fini(PyThreadState *tstate) { - astmodulestate* state = &global_ast_state; +#ifdef Py_BUILD_CORE + struct ast_state *state = &tstate->interp->ast; +#else + struct ast_state *state = &global_ast_state; +#endif + Py_CLEAR(state->AST_type); Py_CLEAR(state->Add_singleton); Py_CLEAR(state->Add_type); @@ -470,7 +486,7 @@ void _PyAST_Fini(PyThreadState *tstate) state->initialized = 0; } -static int init_identifiers(astmodulestate *state) +static int init_identifiers(struct ast_state *state) { if ((state->__dict__ = PyUnicode_InternFromString("__dict__")) == NULL) return 0; if ((state->__doc__ = PyUnicode_InternFromString("__doc__")) == NULL) return 0; @@ -561,7 +577,7 @@ GENERATE_ASDL_SEQ_CONSTRUCTOR(alias, alias_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(withitem, withitem_ty) GENERATE_ASDL_SEQ_CONSTRUCTOR(type_ignore, type_ignore_ty) -static PyObject* ast2obj_mod(astmodulestate *state, void*); +static PyObject* ast2obj_mod(struct ast_state *state, void*); static const char * const Module_fields[]={ "body", "type_ignores", @@ -582,7 +598,7 @@ static const char * const stmt_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_stmt(astmodulestate *state, void*); +static PyObject* ast2obj_stmt(struct ast_state *state, void*); static const char * const FunctionDef_fields[]={ "name", "args", @@ -699,7 +715,7 @@ static const char * const expr_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_expr(astmodulestate *state, void*); +static PyObject* ast2obj_expr(struct ast_state *state, void*); static const char * const BoolOp_fields[]={ "op", "values", @@ -812,12 +828,12 @@ static const char * const Slice_fields[]={ "upper", "step", }; -static PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty); -static PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty); -static PyObject* ast2obj_operator(astmodulestate *state, operator_ty); -static PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty); -static PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty); -static PyObject* ast2obj_comprehension(astmodulestate *state, void*); +static PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty); +static PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty); +static PyObject* ast2obj_operator(struct ast_state *state, operator_ty); +static PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty); +static PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty); +static PyObject* ast2obj_comprehension(struct ast_state *state, void*); static const char * const comprehension_fields[]={ "target", "iter", @@ -830,13 +846,13 @@ static const char * const excepthandler_attributes[] = { "end_lineno", "end_col_offset", }; -static PyObject* ast2obj_excepthandler(astmodulestate *state, void*); +static PyObject* ast2obj_excepthandler(struct ast_state *state, void*); static const char * const ExceptHandler_fields[]={ "type", "name", "body", }; -static PyObject* ast2obj_arguments(astmodulestate *state, void*); +static PyObject* ast2obj_arguments(struct ast_state *state, void*); static const char * const arguments_fields[]={ "posonlyargs", "args", @@ -846,7 +862,7 @@ static const char * const arguments_fields[]={ "kwarg", "defaults", }; -static PyObject* ast2obj_arg(astmodulestate *state, void*); +static PyObject* ast2obj_arg(struct ast_state *state, void*); static const char * const arg_attributes[] = { "lineno", "col_offset", @@ -858,7 +874,7 @@ static const char * const arg_fields[]={ "annotation", "type_comment", }; -static PyObject* ast2obj_keyword(astmodulestate *state, void*); +static PyObject* ast2obj_keyword(struct ast_state *state, void*); static const char * const keyword_attributes[] = { "lineno", "col_offset", @@ -869,17 +885,17 @@ static const char * const keyword_fields[]={ "arg", "value", }; -static PyObject* ast2obj_alias(astmodulestate *state, void*); +static PyObject* ast2obj_alias(struct ast_state *state, void*); static const char * const alias_fields[]={ "name", "asname", }; -static PyObject* ast2obj_withitem(astmodulestate *state, void*); +static PyObject* ast2obj_withitem(struct ast_state *state, void*); static const char * const withitem_fields[]={ "context_expr", "optional_vars", }; -static PyObject* ast2obj_type_ignore(astmodulestate *state, void*); +static PyObject* ast2obj_type_ignore(struct ast_state *state, void*); static const char * const TypeIgnore_fields[]={ "lineno", "tag", @@ -923,7 +939,7 @@ ast_clear(AST_object *self) static int ast_type_init(PyObject *self, PyObject *args, PyObject *kw) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } @@ -999,7 +1015,7 @@ ast_type_init(PyObject *self, PyObject *args, PyObject *kw) static PyObject * ast_type_reduce(PyObject *self, PyObject *unused) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -1054,7 +1070,7 @@ static PyType_Spec AST_type_spec = { }; static PyObject * -make_type(astmodulestate *state, const char *type, PyObject* base, +make_type(struct ast_state *state, const char *type, PyObject* base, const char* const* fields, int num_fields, const char *doc) { PyObject *fnames, *result; @@ -1080,7 +1096,7 @@ make_type(astmodulestate *state, const char *type, PyObject* base, } static int -add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, int num_fields) +add_attributes(struct ast_state *state, PyObject *type, const char * const *attrs, int num_fields) { int i, result; PyObject *s, *l = PyTuple_New(num_fields); @@ -1101,7 +1117,7 @@ add_attributes(astmodulestate *state, PyObject *type, const char * const *attrs, /* Conversion AST -> Python */ -static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (*func)(astmodulestate *state, void*)) +static PyObject* ast2obj_list(struct ast_state *state, asdl_seq *seq, PyObject* (*func)(struct ast_state *state, void*)) { Py_ssize_t i, n = asdl_seq_LEN(seq); PyObject *result = PyList_New(n); @@ -1119,7 +1135,7 @@ static PyObject* ast2obj_list(astmodulestate *state, asdl_seq *seq, PyObject* (* return result; } -static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) +static PyObject* ast2obj_object(struct ast_state *Py_UNUSED(state), void *o) { if (!o) o = Py_None; @@ -1130,14 +1146,14 @@ static PyObject* ast2obj_object(astmodulestate *Py_UNUSED(state), void *o) #define ast2obj_identifier ast2obj_object #define ast2obj_string ast2obj_object -static PyObject* ast2obj_int(astmodulestate *Py_UNUSED(state), long b) +static PyObject* ast2obj_int(struct ast_state *Py_UNUSED(state), long b) { return PyLong_FromLong(b); } /* Conversion Python -> AST */ -static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_object(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (obj == Py_None) obj = NULL; @@ -1152,7 +1168,7 @@ static int obj2ast_object(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObj return 0; } -static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_constant(struct ast_state *Py_UNUSED(state), PyObject* obj, PyObject** out, PyArena* arena) { if (PyArena_AddPyObject(arena, obj) < 0) { *out = NULL; @@ -1163,7 +1179,7 @@ static int obj2ast_constant(astmodulestate *Py_UNUSED(state), PyObject* obj, PyO return 0; } -static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_identifier(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && obj != Py_None) { PyErr_SetString(PyExc_TypeError, "AST identifier must be of type str"); @@ -1172,7 +1188,7 @@ static int obj2ast_identifier(astmodulestate *state, PyObject* obj, PyObject** o return obj2ast_object(state, obj, out, arena); } -static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, PyArena* arena) +static int obj2ast_string(struct ast_state *state, PyObject* obj, PyObject** out, PyArena* arena) { if (!PyUnicode_CheckExact(obj) && !PyBytes_CheckExact(obj)) { PyErr_SetString(PyExc_TypeError, "AST string must be of type str"); @@ -1181,7 +1197,7 @@ static int obj2ast_string(astmodulestate *state, PyObject* obj, PyObject** out, return obj2ast_object(state, obj, out, arena); } -static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) +static int obj2ast_int(struct ast_state* Py_UNUSED(state), PyObject* obj, int* out, PyArena* arena) { int i; if (!PyLong_Check(obj)) { @@ -1196,7 +1212,7 @@ static int obj2ast_int(astmodulestate* Py_UNUSED(state), PyObject* obj, int* out return 0; } -static int add_ast_fields(astmodulestate *state) +static int add_ast_fields(struct ast_state *state) { PyObject *empty_tuple; empty_tuple = PyTuple_New(0); @@ -1211,7 +1227,7 @@ static int add_ast_fields(astmodulestate *state) } -static int init_types(astmodulestate *state) +static int init_types(struct ast_state *state) { if (state->initialized) return 1; if (init_identifiers(state) < 0) return 0; @@ -1890,37 +1906,37 @@ static int init_types(astmodulestate *state) return 1; } -static int obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, +static int obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena); -static int obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, +static int obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* arena); -static int obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, +static int obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* arena); -static int obj2ast_expr_context(astmodulestate *state, PyObject* obj, +static int obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* out, PyArena* arena); -static int obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, - PyArena* arena); -static int obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* - out, PyArena* arena); -static int obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* +static int obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* + out, PyArena* arena); +static int obj2ast_operator(struct ast_state *state, PyObject* obj, + operator_ty* out, PyArena* arena); +static int obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, PyArena* arena); -static int obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, +static int obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* arena); -static int obj2ast_comprehension(astmodulestate *state, PyObject* obj, +static int obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* out, PyArena* arena); -static int obj2ast_excepthandler(astmodulestate *state, PyObject* obj, +static int obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* out, PyArena* arena); -static int obj2ast_arguments(astmodulestate *state, PyObject* obj, +static int obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, PyArena* arena); -static int obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, +static int obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena); -static int obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* +static int obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, PyArena* arena); -static int obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, +static int obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* arena); -static int obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* - out, PyArena* arena); -static int obj2ast_type_ignore(astmodulestate *state, PyObject* obj, +static int obj2ast_withitem(struct ast_state *state, PyObject* obj, + withitem_ty* out, PyArena* arena); +static int obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* out, PyArena* arena); mod_ty @@ -3391,7 +3407,7 @@ TypeIgnore(int lineno, string tag, PyArena *arena) PyObject* -ast2obj_mod(astmodulestate *state, void* _o) +ast2obj_mod(struct ast_state *state, void* _o) { mod_ty o = (mod_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3462,7 +3478,7 @@ failed: } PyObject* -ast2obj_stmt(astmodulestate *state, void* _o) +ast2obj_stmt(struct ast_state *state, void* _o) { stmt_ty o = (stmt_ty)_o; PyObject *result = NULL, *value = NULL; @@ -3977,7 +3993,7 @@ failed: } PyObject* -ast2obj_expr(astmodulestate *state, void* _o) +ast2obj_expr(struct ast_state *state, void* _o) { expr_ty o = (expr_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4453,7 +4469,7 @@ failed: return NULL; } -PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) +PyObject* ast2obj_expr_context(struct ast_state *state, expr_context_ty o) { switch(o) { case Load: @@ -4468,7 +4484,7 @@ PyObject* ast2obj_expr_context(astmodulestate *state, expr_context_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) +PyObject* ast2obj_boolop(struct ast_state *state, boolop_ty o) { switch(o) { case And: @@ -4480,7 +4496,7 @@ PyObject* ast2obj_boolop(astmodulestate *state, boolop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) +PyObject* ast2obj_operator(struct ast_state *state, operator_ty o) { switch(o) { case Add: @@ -4525,7 +4541,7 @@ PyObject* ast2obj_operator(astmodulestate *state, operator_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) +PyObject* ast2obj_unaryop(struct ast_state *state, unaryop_ty o) { switch(o) { case Invert: @@ -4543,7 +4559,7 @@ PyObject* ast2obj_unaryop(astmodulestate *state, unaryop_ty o) } Py_UNREACHABLE(); } -PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) +PyObject* ast2obj_cmpop(struct ast_state *state, cmpop_ty o) { switch(o) { case Eq: @@ -4580,7 +4596,7 @@ PyObject* ast2obj_cmpop(astmodulestate *state, cmpop_ty o) Py_UNREACHABLE(); } PyObject* -ast2obj_comprehension(astmodulestate *state, void* _o) +ast2obj_comprehension(struct ast_state *state, void* _o) { comprehension_ty o = (comprehension_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4619,7 +4635,7 @@ failed: } PyObject* -ast2obj_excepthandler(astmodulestate *state, void* _o) +ast2obj_excepthandler(struct ast_state *state, void* _o) { excepthandler_ty o = (excepthandler_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4678,7 +4694,7 @@ failed: } PyObject* -ast2obj_arguments(astmodulestate *state, void* _o) +ast2obj_arguments(struct ast_state *state, void* _o) { arguments_ty o = (arguments_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4732,7 +4748,7 @@ failed: } PyObject* -ast2obj_arg(astmodulestate *state, void* _o) +ast2obj_arg(struct ast_state *state, void* _o) { arg_ty o = (arg_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4786,7 +4802,7 @@ failed: } PyObject* -ast2obj_keyword(astmodulestate *state, void* _o) +ast2obj_keyword(struct ast_state *state, void* _o) { keyword_ty o = (keyword_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4835,7 +4851,7 @@ failed: } PyObject* -ast2obj_alias(astmodulestate *state, void* _o) +ast2obj_alias(struct ast_state *state, void* _o) { alias_ty o = (alias_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4864,7 +4880,7 @@ failed: } PyObject* -ast2obj_withitem(astmodulestate *state, void* _o) +ast2obj_withitem(struct ast_state *state, void* _o) { withitem_ty o = (withitem_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4893,7 +4909,7 @@ failed: } PyObject* -ast2obj_type_ignore(astmodulestate *state, void* _o) +ast2obj_type_ignore(struct ast_state *state, void* _o) { type_ignore_ty o = (type_ignore_ty)_o; PyObject *result = NULL, *value = NULL; @@ -4927,7 +4943,7 @@ failed: int -obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) +obj2ast_mod(struct ast_state *state, PyObject* obj, mod_ty* out, PyArena* arena) { int isinstance; @@ -5154,7 +5170,8 @@ obj2ast_mod(astmodulestate *state, PyObject* obj, mod_ty* out, PyArena* arena) } int -obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) +obj2ast_stmt(struct ast_state *state, PyObject* obj, stmt_ty* out, PyArena* + arena) { int isinstance; @@ -7052,7 +7069,8 @@ obj2ast_stmt(astmodulestate *state, PyObject* obj, stmt_ty* out, PyArena* arena) } int -obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) +obj2ast_expr(struct ast_state *state, PyObject* obj, expr_ty* out, PyArena* + arena) { int isinstance; @@ -8562,7 +8580,7 @@ obj2ast_expr(astmodulestate *state, PyObject* obj, expr_ty* out, PyArena* arena) } int -obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* +obj2ast_expr_context(struct ast_state *state, PyObject* obj, expr_context_ty* out, PyArena* arena) { int isinstance; @@ -8597,7 +8615,7 @@ obj2ast_expr_context(astmodulestate *state, PyObject* obj, expr_context_ty* } int -obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* +obj2ast_boolop(struct ast_state *state, PyObject* obj, boolop_ty* out, PyArena* arena) { int isinstance; @@ -8624,7 +8642,7 @@ obj2ast_boolop(astmodulestate *state, PyObject* obj, boolop_ty* out, PyArena* } int -obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, +obj2ast_operator(struct ast_state *state, PyObject* obj, operator_ty* out, PyArena* arena) { int isinstance; @@ -8739,8 +8757,8 @@ obj2ast_operator(astmodulestate *state, PyObject* obj, operator_ty* out, } int -obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* - arena) +obj2ast_unaryop(struct ast_state *state, PyObject* obj, unaryop_ty* out, + PyArena* arena) { int isinstance; @@ -8782,7 +8800,7 @@ obj2ast_unaryop(astmodulestate *state, PyObject* obj, unaryop_ty* out, PyArena* } int -obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* +obj2ast_cmpop(struct ast_state *state, PyObject* obj, cmpop_ty* out, PyArena* arena) { int isinstance; @@ -8873,7 +8891,7 @@ obj2ast_cmpop(astmodulestate *state, PyObject* obj, cmpop_ty* out, PyArena* } int -obj2ast_comprehension(astmodulestate *state, PyObject* obj, comprehension_ty* +obj2ast_comprehension(struct ast_state *state, PyObject* obj, comprehension_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -8962,7 +8980,7 @@ failed: } int -obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* +obj2ast_excepthandler(struct ast_state *state, PyObject* obj, excepthandler_ty* out, PyArena* arena) { int isinstance; @@ -9112,7 +9130,7 @@ obj2ast_excepthandler(astmodulestate *state, PyObject* obj, excepthandler_ty* } int -obj2ast_arguments(astmodulestate *state, PyObject* obj, arguments_ty* out, +obj2ast_arguments(struct ast_state *state, PyObject* obj, arguments_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9324,7 +9342,7 @@ failed: } int -obj2ast_arg(astmodulestate *state, PyObject* obj, arg_ty* out, PyArena* arena) +obj2ast_arg(struct ast_state *state, PyObject* obj, arg_ty* out, PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -9435,8 +9453,8 @@ failed: } int -obj2ast_keyword(astmodulestate *state, PyObject* obj, keyword_ty* out, PyArena* - arena) +obj2ast_keyword(struct ast_state *state, PyObject* obj, keyword_ty* out, + PyArena* arena) { PyObject* tmp = NULL; identifier arg; @@ -9533,7 +9551,7 @@ failed: } int -obj2ast_alias(astmodulestate *state, PyObject* obj, alias_ty* out, PyArena* +obj2ast_alias(struct ast_state *state, PyObject* obj, alias_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9574,7 +9592,7 @@ failed: } int -obj2ast_withitem(astmodulestate *state, PyObject* obj, withitem_ty* out, +obj2ast_withitem(struct ast_state *state, PyObject* obj, withitem_ty* out, PyArena* arena) { PyObject* tmp = NULL; @@ -9615,8 +9633,8 @@ failed: } int -obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, - PyArena* arena) +obj2ast_type_ignore(struct ast_state *state, PyObject* obj, type_ignore_ty* + out, PyArena* arena) { int isinstance; @@ -9677,9 +9695,8 @@ obj2ast_type_ignore(astmodulestate *state, PyObject* obj, type_ignore_ty* out, static int astmodule_exec(PyObject *m) { - astmodulestate *state = get_ast_state(m); - - if (!init_types(state)) { + struct ast_state *state = get_ast_state(); + if (state == NULL) { return -1; } if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { @@ -10132,7 +10149,7 @@ static PyModuleDef_Slot astmodule_slots[] = { static struct PyModuleDef _astmodule = { PyModuleDef_HEAD_INIT, .m_name = "_ast", - // The _ast module uses a global state (global_ast_state). + // The _ast module uses a per-interpreter state (PyInterpreterState.ast) .m_size = 0, .m_slots = astmodule_slots, }; @@ -10146,7 +10163,7 @@ PyInit__ast(void) PyObject* PyAST_mod2obj(mod_ty t) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return NULL; } @@ -10163,7 +10180,11 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) return NULL; } - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); + if (state == NULL) { + return NULL; + } + PyObject *req_type[3]; req_type[0] = state->Module_type; req_type[1] = state->Expression_type; @@ -10189,7 +10210,7 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode) int PyAST_Check(PyObject* obj) { - astmodulestate *state = get_global_ast_state(); + struct ast_state *state = get_ast_state(); if (state == NULL) { return -1; } From 45df61fd2d58e8db33179f3b5d00e53fe6a7e592 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 2 Nov 2020 23:17:46 +0100 Subject: [PATCH 08/61] bpo-26789: Fix logging.FileHandler._open() at exit (GH-23053) The logging.FileHandler class now keeps a reference to the builtin open() function to be able to open or reopen the file during Python finalization. Fix errors like: Exception ignored in: (...) Traceback (most recent call last): (...) File ".../logging/__init__.py", line 1463, in error File ".../logging/__init__.py", line 1577, in _log File ".../logging/__init__.py", line 1587, in handle File ".../logging/__init__.py", line 1649, in callHandlers File ".../logging/__init__.py", line 948, in handle File ".../logging/__init__.py", line 1182, in emit File ".../logging/__init__.py", line 1171, in _open NameError: name 'open' is not defined --- Lib/logging/__init__.py | 9 +++- Lib/test/test_logging.py | 43 ++++++++++++++++--- .../2020-10-31-01-16-49.bpo-26789.9BdNAt.rst | 4 ++ Python/_warnings.c | 2 +- 4 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 265e286101e..badfd654b16 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -1148,6 +1148,10 @@ class FileHandler(StreamHandler): self.encoding = encoding self.errors = errors self.delay = delay + # bpo-26789: FileHandler keeps a reference to the builtin open() + # function to be able to open or reopen the file during Python + # finalization. + self._builtin_open = open if delay: #We don't open the stream, but we still need to call the #Handler constructor to set level, formatter, lock etc. @@ -1183,8 +1187,9 @@ class FileHandler(StreamHandler): Open the current base file with the (original) mode and encoding. Return the resulting stream. """ - return open(self.baseFilename, self.mode, encoding=self.encoding, - errors=self.errors) + open_func = self._builtin_open + return open_func(self.baseFilename, self.mode, + encoding=self.encoding, errors=self.errors) def emit(self, record): """ diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 7c98e19b740..e2196736dcd 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -4310,8 +4310,8 @@ class ModuleLevelMiscTest(BaseTest): logging.setLoggerClass(logging.Logger) def test_logging_at_shutdown(self): - # Issue #20037 - code = """if 1: + # bpo-20037: Doing text I/O late at interpreter shutdown must not crash + code = textwrap.dedent(""" import logging class A: @@ -4321,22 +4321,55 @@ class ModuleLevelMiscTest(BaseTest): except Exception: logging.exception("exception in __del__") - a = A()""" + a = A() + """) rc, out, err = assert_python_ok("-c", code) err = err.decode() self.assertIn("exception in __del__", err) self.assertIn("ValueError: some error", err) + def test_logging_at_shutdown_open(self): + # bpo-26789: FileHandler keeps a reference to the builtin open() + # function to be able to open or reopen the file during Python + # finalization. + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + + code = textwrap.dedent(f""" + import builtins + import logging + + class A: + def __del__(self): + logging.error("log in __del__") + + # basicConfig() opens the file, but logging.shutdown() closes + # it at Python exit. When A.__del__() is called, + # FileHandler._open() must be called again to re-open the file. + logging.basicConfig(filename={filename!r}) + + a = A() + + # Simulate the Python finalization which removes the builtin + # open() function. + del builtins.open + """) + assert_python_ok("-c", code) + + with open(filename) as fp: + self.assertEqual(fp.read().rstrip(), "ERROR:root:log in __del__") + def test_recursion_error(self): # Issue 36272 - code = """if 1: + code = textwrap.dedent(""" import logging def rec(): logging.error("foo") rec() - rec()""" + rec() + """) rc, out, err = assert_python_failure("-c", code) err = err.decode() self.assertNotIn("Cannot recover from stack overflow.", err) diff --git a/Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst b/Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst new file mode 100644 index 00000000000..d8832401c93 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst @@ -0,0 +1,4 @@ +The :class:`logging.FileHandler` class now keeps a reference to the builtin +:func:`open` function to be able to open or reopen the file during Python +finalization. Fix errors like: ``NameError: name 'open' is not defined``. Patch +by Victor Stinner. diff --git a/Python/_warnings.c b/Python/_warnings.c index 271cd47f4ee..3c048af4193 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -852,7 +852,7 @@ setup_context(Py_ssize_t stack_level, PyObject **filename, int *lineno, } if (f == NULL) { - globals = _PyInterpreterState_GET()->sysdict; + globals = tstate->interp->sysdict; *filename = PyUnicode_FromString("sys"); *lineno = 1; } From 9568622c9983b682b2a2a7bacfd3c341028ea099 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Kul=C3=ADk?= Date: Tue, 3 Nov 2020 00:10:01 +0100 Subject: [PATCH 09/61] bpo-35455: Fix thread_time for Solaris OS (GH-11118) --- .../2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst | 3 +++ Modules/timemodule.c | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst diff --git a/Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst b/Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst new file mode 100644 index 00000000000..e72c7d277a1 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst @@ -0,0 +1,3 @@ +On Solaris, :func:`~time.thread_time` is now implemented with +``gethrvtime()`` because ``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` is not +always available. Patch by Jakub Kulik. diff --git a/Modules/timemodule.c b/Modules/timemodule.c index 8a4d149befb..eb192c5e7fd 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -1371,6 +1371,23 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) return 0; } +#elif defined(__sun) && defined(__SVR4) +#define HAVE_THREAD_TIME +static int +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) +{ + /* bpo-35455: On Solaris, CLOCK_THREAD_CPUTIME_ID clock is not always + available; use gethrvtime() to substitute this functionality. */ + if (info) { + info->implementation = "gethrvtime()"; + info->resolution = 1e-9; + info->monotonic = 1; + info->adjustable = 0; + } + *tp = _PyTime_FromNanoseconds(gethrvtime()); + return 0; +} + #elif defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) #define HAVE_THREAD_TIME static int From 114ee5dec0ba2163947833e793ece4173108e0d2 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 3 Nov 2020 00:00:12 +0000 Subject: [PATCH 10/61] Python 3.10.0a2 --- Include/patchlevel.h | 4 +- Lib/pydoc_data/topics.py | 198 ++-- Misc/NEWS.d/3.10.0a2.rst | 912 ++++++++++++++++++ .../2020-10-21-18-31-54.bpo-38980.xz7BNd.rst | 3 - .../2020-09-14-10-17-00.bpo-41784.Yl4gI2.rst | 1 - .../2020-09-28-14-31-07.bpo-41756.ZZ5wJG.rst | 3 - .../2020-10-09-22-50-46.bpo-41986.JUPE59.rst | 2 - .../2020-10-10-14-05-24.bpo-40422.sh8IDY.rst | 1 - .../2020-10-11-05-05-53.bpo-41845.ZFvuQM.rst | 2 - .../2020-10-11-19-17-44.bpo-40423.GsmgEj.rst | 3 - .../2020-10-12-20-13-58.bpo-42015.X4H2_V.rst | 3 - .../2020-10-16-10-47-17.bpo-42157.e3BcPM.rst | 3 - .../2020-05-27-16-08-16.bpo-38605.rcs2uK.rst | 3 - .../2020-08-07-13-42-48.bpo-41435.qPWjJA.rst | 1 - .../2020-10-01-22-44-23.bpo-41902.iLoMVF.rst | 1 - .../2020-10-02-11-35-33.bpo-41894.ffmtOt.rst | 3 - .../2020-10-02-13-32-05.bpo-41902.ZKTxzW.rst | 3 - .../2020-10-08-09-58-19.bpo-41974.8B-q8O.rst | 4 - .../2020-10-09-10-55-50.bpo-41979.ImXIk2.rst | 1 - .../2020-10-10-13-53-52.bpo-41993.YMzixQ.rst | 2 - .../2020-10-14-16-19-43.bpo-41984.SEtKMr.rst | 2 - .../2020-10-15-21-55-32.bpo-42030.PmU2CA.rst | 3 - .../2020-10-20-04-24-07.bpo-42093.ooZZNh.rst | 2 - .../2020-10-21-14-40-54.bpo-41910.CzBMit.rst | 1 - .../2020-10-23-02-43-24.bpo-42123.64gJWC.rst | 3 - .../2020-10-25-21-14-18.bpo-42150.b70u_T.rst | 2 - .../2020-10-27-18-32-49.bpo-41659.d4a-8o.rst | 3 - .../2020-10-27-21-34-05.bpo-42143.N6KXUO.rst | 2 - .../2020-10-29-12-49-08.bpo-41796.tkGdHq.rst | 2 - .../2020-10-30-13-11-01.bpo-42206.xxssR8.rst | 2 - .../2020-10-30-22-16-30.bpo-42214.lXskM_.rst | 2 - .../2020-10-31-17-50-23.bpo-42218.Dp_Z3v.rst | 3 - .../2020-11-01-21-21-38.bpo-42236.MPx-NK.rst | 2 - .../2020-01-22-05-14-53.bpo-39416.uYjhEm.rst | 1 - .../2020-02-24-09-02-05.bpo-39693.QXw0Fm.rst | 1 - .../2020-10-10-01-36-37.bpo-41805.l-CGv5.rst | 3 - .../2020-10-21-02-21-14.bpo-42010.76vJ0u.rst | 4 - .../2020-10-28-21-39-45.bpo-42061._x-0sg.rst | 1 - .../2020-04-22-09-37-40.bpo-38439.ieXL-c.rst | 2 - .../2020-06-16-12-16-13.bpo-40511.XkihpM.rst | 3 - .../2020-10-24-21-27-37.bpo-33987.fIh9JL.rst | 3 - .../2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst | 4 - .../2020-01-19-18-40-26.bpo-27321.8e6SpM.rst | 2 - .../2020-04-21-17-18-33.bpo-34204.9wXTtY.rst | 2 - .../2020-05-04-12-16-00.bpo-40492.ONk9Na.rst | 3 - .../2020-05-14-16-01-34.bpo-40592.Cmk855.rst | 1 - .../2020-05-31-10-48-47.bpo-4356.P8kXqp.rst | 1 - .../2020-07-08-09-45-00.bpo-16936.z8o8Pn.rst | 1 - .../2020-07-19-20-10-41.bpo-41229.p8rJa2.rst | 3 - .../2020-08-19-08-32-13.bpo-41586.IYjmjK.rst | 2 - .../2020-09-04-17-33-04.bpo-30681.LR4fnY.rst | 2 - .../2020-09-08-23-41-29.bpo-41747.M6wLKv.rst | 3 - .../2020-09-13-02-02-18.bpo-39337.L3NXTt.rst | 1 - .../2020-09-22-11-07-50.bpo-41831.k-Eop_.rst | 3 - .../2020-09-29-16-23-54.bpo-41876.QicdDU.rst | 1 - .../2020-09-30-11-05-11.bpo-23706.dHTGjF.rst | 1 - .../2020-10-01-21-11-03.bpo-41905._JpjR4.rst | 1 - .../2020-10-03-23-14-50.bpo-41923.Buonw9.rst | 1 - .../2020-10-07-18-36-03.bpo-41943.Pt55fT.rst | 1 - .../2020-10-08-18-22-28.bpo-41976.Svm0wb.rst | 3 - .../2020-10-11-21-43-03.bpo-39101.-I49Pm.rst | 1 - .../2020-10-12-21-21-24.bpo-42021.8yv_8-.rst | 1 - .../2020-10-15-17-20-37.bpo-42043.OS0p_v.rst | 4 - .../2020-10-16-07-45-35.bpo-35823.SNQo56.rst | 2 - .../2020-10-16-16-08-04.bpo-28660.eX9pvD.rst | 1 - .../2020-10-16-22-48-01.bpo-19270.jd_gkA.rst | 2 - .../2020-10-17-07-52-53.bpo-41966.gwEQRZ.rst | 2 - .../2020-10-17-23-17-18.bpo-42065.85BsRA.rst | 3 - .../2020-10-19-14-02-09.bpo-41491.d1BUWH.rst | 1 - .../2020-10-19-16-53-19.bpo-42089.R1dthW.rst | 2 - .../2020-10-21-23-45-02.bpo-41052.3N7J2J.rst | 2 - .../2020-10-23-08-54-47.bpo-41490.-Yk6OD.rst | 3 - .../2020-10-23-19-20-14.bpo-42103.C5obK2.rst | 3 - .../2020-10-25-19-25-02.bpo-42146.6A8uvS.rst | 2 - .../2020-10-26-19-08-07.bpo-42157.Bdpa04.rst | 4 - .../2020-10-26-23-29-16.bpo-42157.4wuwTe.rst | 3 - .../2020-10-31-01-16-49.bpo-26789.9BdNAt.rst | 4 - .../2020-10-31-13-28-36.bpo-29566.6aDbty.rst | 1 - .../2020-11-02-01-31-15.bpo-42233.YxRj-h.rst | 3 - .../2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst | 3 - .../2020-10-19-10-56-27.bpo-42051.EU_B7u.rst | 3 - .../2020-10-23-19-19-30.bpo-42103.cILT66.rst | 2 - .../2020-08-03-13-44-37.bpo-41306.VDoWXI.rst | 1 - .../2020-10-05-17-43-46.bpo-41944.rf1dYb.rst | 1 - .../2020-10-08-14-00-17.bpo-41970.aZ8QFf.rst | 2 - .../2020-10-12-00-11-47.bpo-41739.wSCc4K.rst | 2 - .../2020-05-30-02-46-43.bpo-38324.476M-5.rst | 1 - .../2020-08-26-09-35-06.bpo-41557.vt00cQ.rst | 1 - .../2020-09-24-23-09-40.bpo-39107.GbUZvD.rst | 1 - .../2020-10-18-18-43-45.bpo-38252.7Nlepg.rst | 1 - .../2020-10-20-13-19-42.bpo-38439.eMLi-t.rst | 1 - .../2020-10-19-12-25-19.bpo-41471.gwA7un.rst | 1 - README.rst | 2 +- 93 files changed, 1047 insertions(+), 255 deletions(-) create mode 100644 Misc/NEWS.d/3.10.0a2.rst delete mode 100644 Misc/NEWS.d/next/Build/2020-10-21-18-31-54.bpo-38980.xz7BNd.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-09-14-10-17-00.bpo-41784.Yl4gI2.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-09-28-14-31-07.bpo-41756.ZZ5wJG.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-10-09-22-50-46.bpo-41986.JUPE59.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-10-10-14-05-24.bpo-40422.sh8IDY.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-10-11-05-05-53.bpo-41845.ZFvuQM.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-10-11-19-17-44.bpo-40423.GsmgEj.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-10-12-20-13-58.bpo-42015.X4H2_V.rst delete mode 100644 Misc/NEWS.d/next/C API/2020-10-16-10-47-17.bpo-42157.e3BcPM.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-05-27-16-08-16.bpo-38605.rcs2uK.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-01-22-44-23.bpo-41902.iLoMVF.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-02-11-35-33.bpo-41894.ffmtOt.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-02-13-32-05.bpo-41902.ZKTxzW.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-08-09-58-19.bpo-41974.8B-q8O.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-09-10-55-50.bpo-41979.ImXIk2.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-10-13-53-52.bpo-41993.YMzixQ.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-14-16-19-43.bpo-41984.SEtKMr.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-15-21-55-32.bpo-42030.PmU2CA.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-20-04-24-07.bpo-42093.ooZZNh.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-21-14-40-54.bpo-41910.CzBMit.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-23-02-43-24.bpo-42123.64gJWC.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-25-21-14-18.bpo-42150.b70u_T.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-27-18-32-49.bpo-41659.d4a-8o.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-27-21-34-05.bpo-42143.N6KXUO.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-30-13-11-01.bpo-42206.xxssR8.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-30-22-16-30.bpo-42214.lXskM_.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-10-31-17-50-23.bpo-42218.Dp_Z3v.rst delete mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-01-21-21-38.bpo-42236.MPx-NK.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2020-01-22-05-14-53.bpo-39416.uYjhEm.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2020-02-24-09-02-05.bpo-39693.QXw0Fm.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2020-10-10-01-36-37.bpo-41805.l-CGv5.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2020-10-21-02-21-14.bpo-42010.76vJ0u.rst delete mode 100644 Misc/NEWS.d/next/Documentation/2020-10-28-21-39-45.bpo-42061._x-0sg.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2020-04-22-09-37-40.bpo-38439.ieXL-c.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2020-06-16-12-16-13.bpo-40511.XkihpM.rst delete mode 100644 Misc/NEWS.d/next/IDLE/2020-10-24-21-27-37.bpo-33987.fIh9JL.rst delete mode 100644 Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-01-19-18-40-26.bpo-27321.8e6SpM.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-04-21-17-18-33.bpo-34204.9wXTtY.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-05-04-12-16-00.bpo-40492.ONk9Na.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-05-14-16-01-34.bpo-40592.Cmk855.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-05-31-10-48-47.bpo-4356.P8kXqp.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-07-08-09-45-00.bpo-16936.z8o8Pn.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-08-19-08-32-13.bpo-41586.IYjmjK.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-09-04-17-33-04.bpo-30681.LR4fnY.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-09-08-23-41-29.bpo-41747.M6wLKv.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-09-13-02-02-18.bpo-39337.L3NXTt.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-09-22-11-07-50.bpo-41831.k-Eop_.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-09-29-16-23-54.bpo-41876.QicdDU.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-09-30-11-05-11.bpo-23706.dHTGjF.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-01-21-11-03.bpo-41905._JpjR4.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-03-23-14-50.bpo-41923.Buonw9.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-11-21-43-03.bpo-39101.-I49Pm.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-12-21-21-24.bpo-42021.8yv_8-.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-15-17-20-37.bpo-42043.OS0p_v.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-16-16-08-04.bpo-28660.eX9pvD.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-16-22-48-01.bpo-19270.jd_gkA.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-17-07-52-53.bpo-41966.gwEQRZ.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-17-23-17-18.bpo-42065.85BsRA.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-19-14-02-09.bpo-41491.d1BUWH.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-19-16-53-19.bpo-42089.R1dthW.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-23-08-54-47.bpo-41490.-Yk6OD.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-26-19-08-07.bpo-42157.Bdpa04.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-26-23-29-16.bpo-42157.4wuwTe.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-10-31-13-28-36.bpo-29566.6aDbty.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-11-02-01-31-15.bpo-42233.YxRj-h.rst delete mode 100644 Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst delete mode 100644 Misc/NEWS.d/next/Security/2020-10-19-10-56-27.bpo-42051.EU_B7u.rst delete mode 100644 Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst delete mode 100644 Misc/NEWS.d/next/Tests/2020-08-03-13-44-37.bpo-41306.VDoWXI.rst delete mode 100644 Misc/NEWS.d/next/Tests/2020-10-05-17-43-46.bpo-41944.rf1dYb.rst delete mode 100644 Misc/NEWS.d/next/Tests/2020-10-08-14-00-17.bpo-41970.aZ8QFf.rst delete mode 100644 Misc/NEWS.d/next/Tests/2020-10-12-00-11-47.bpo-41739.wSCc4K.rst delete mode 100644 Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst delete mode 100644 Misc/NEWS.d/next/Windows/2020-08-26-09-35-06.bpo-41557.vt00cQ.rst delete mode 100644 Misc/NEWS.d/next/Windows/2020-09-24-23-09-40.bpo-39107.GbUZvD.rst delete mode 100644 Misc/NEWS.d/next/Windows/2020-10-18-18-43-45.bpo-38252.7Nlepg.rst delete mode 100644 Misc/NEWS.d/next/Windows/2020-10-20-13-19-42.bpo-38439.eMLi-t.rst delete mode 100644 Misc/NEWS.d/next/macOS/2020-10-19-12-25-19.bpo-41471.gwA7un.rst diff --git a/Include/patchlevel.h b/Include/patchlevel.h index eeb23af9766..b9bdc598ebc 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -20,10 +20,10 @@ #define PY_MINOR_VERSION 10 #define PY_MICRO_VERSION 0 #define PY_RELEASE_LEVEL PY_RELEASE_LEVEL_ALPHA -#define PY_RELEASE_SERIAL 1 +#define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.10.0a1+" +#define PY_VERSION "3.10.0a2" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. diff --git a/Lib/pydoc_data/topics.py b/Lib/pydoc_data/topics.py index 1fdb1ae859e..b424b1d0f9b 100644 --- a/Lib/pydoc_data/topics.py +++ b/Lib/pydoc_data/topics.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Autogenerated by Sphinx on Mon Oct 5 18:27:28 2020 +# Autogenerated by Sphinx on Tue Nov 3 00:01:01 2020 topics = {'assert': 'The "assert" statement\n' '**********************\n' '\n' @@ -433,11 +433,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'Execution of Python coroutines can be suspended and resumed at ' 'many\n' - 'points (see *coroutine*). Inside the body of a coroutine ' - 'function,\n' - '"await" and "async" identifiers become reserved keywords; "await"\n' - 'expressions, "async for" and "async with" can only be used in\n' - 'coroutine function bodies.\n' + 'points (see *coroutine*). "await" expressions, "async for" and ' + '"async\n' + 'with" can only be used in the body of a coroutine function.\n' '\n' 'Functions defined with "async def" syntax are always coroutine\n' 'functions, even if they do not contain "await" or "async" ' @@ -453,6 +451,10 @@ topics = {'assert': 'The "assert" statement\n' ' do_stuff()\n' ' await some_coroutine()\n' '\n' + 'Changed in version 3.7: "await" and "async" are now keywords;\n' + 'previously they were only treated as such inside the body of a\n' + 'coroutine function.\n' + '\n' '\n' 'The "async for" statement\n' '=========================\n' @@ -700,6 +702,11 @@ topics = {'assert': 'The "assert" statement\n' 'syntax or\n' ' built-in functions. See Special method lookup.\n' '\n' + ' For certain sensitive attribute accesses, raises an ' + 'auditing event\n' + ' "object.__getattr__" with arguments "obj" and ' + '"name".\n' + '\n' 'object.__setattr__(self, name, value)\n' '\n' ' Called when an attribute assignment is attempted. ' @@ -716,6 +723,11 @@ topics = {'assert': 'The "assert" statement\n' 'for example,\n' ' "object.__setattr__(self, name, value)".\n' '\n' + ' For certain sensitive attribute assignments, raises ' + 'an auditing\n' + ' event "object.__setattr__" with arguments "obj", ' + '"name", "value".\n' + '\n' 'object.__delattr__(self, name)\n' '\n' ' Like "__setattr__()" but for attribute deletion ' @@ -724,6 +736,11 @@ topics = {'assert': 'The "assert" statement\n' 'obj.name" is\n' ' meaningful for the object.\n' '\n' + ' For certain sensitive attribute deletions, raises an ' + 'auditing event\n' + ' "object.__delattr__" with arguments "obj" and ' + '"name".\n' + '\n' 'object.__dir__(self)\n' '\n' ' Called when "dir()" is called on the object. A ' @@ -1464,8 +1481,8 @@ topics = {'assert': 'The "assert" statement\n' '\n' ' Called when the instance is “called” as a function; if ' 'this method\n' - ' is defined, "x(arg1, arg2, ...)" is a shorthand for\n' - ' "x.__call__(arg1, arg2, ...)".\n', + ' is defined, "x(arg1, arg2, ...)" roughly translates to\n' + ' "type(x).__call__(x, arg1, ...)".\n', 'calls': 'Calls\n' '*****\n' '\n' @@ -2766,20 +2783,11 @@ topics = {'assert': 'The "assert" statement\n' 'parameter list. These annotations can be any valid Python ' 'expression.\n' 'The presence of annotations does not change the semantics of a\n' - 'function. The annotation values are available as values of a\n' + 'function. The annotation values are available as string values ' + 'in a\n' 'dictionary keyed by the parameters’ names in the ' '"__annotations__"\n' - 'attribute of the function object. If the "annotations" import ' - 'from\n' - '"__future__" is used, annotations are preserved as strings at ' - 'runtime\n' - 'which enables postponed evaluation. Otherwise, they are ' - 'evaluated\n' - 'when the function definition is executed. In this case ' - 'annotations\n' - 'may be evaluated in a different order than they appear in the ' - 'source\n' - 'code.\n' + 'attribute of the function object.\n' '\n' 'It is also possible to create anonymous functions (functions not ' 'bound\n' @@ -2949,12 +2957,9 @@ topics = {'assert': 'The "assert" statement\n' '\n' 'Execution of Python coroutines can be suspended and resumed at ' 'many\n' - 'points (see *coroutine*). Inside the body of a coroutine ' - 'function,\n' - '"await" and "async" identifiers become reserved keywords; ' - '"await"\n' - 'expressions, "async for" and "async with" can only be used in\n' - 'coroutine function bodies.\n' + 'points (see *coroutine*). "await" expressions, "async for" and ' + '"async\n' + 'with" can only be used in the body of a coroutine function.\n' '\n' 'Functions defined with "async def" syntax are always coroutine\n' 'functions, even if they do not contain "await" or "async" ' @@ -2970,6 +2975,10 @@ topics = {'assert': 'The "assert" statement\n' ' do_stuff()\n' ' await some_coroutine()\n' '\n' + 'Changed in version 3.7: "await" and "async" are now keywords;\n' + 'previously they were only treated as such inside the body of a\n' + 'coroutine function.\n' + '\n' '\n' 'The "async for" statement\n' '-------------------------\n' @@ -3461,16 +3470,21 @@ topics = {'assert': 'The "assert" statement\n' ' on the value to determine if the result is true or ' 'false.\n' '\n' - ' By default, "__ne__()" delegates to "__eq__()" and ' - 'inverts the\n' - ' result unless it is "NotImplemented". There are no ' - 'other implied\n' - ' relationships among the comparison operators, for ' - 'example, the\n' - ' truth of "(x` previously did not show the parameterized types in the +``GenericAlias``. They have now been changed to do so. + +.. + +.. bpo: 29566 +.. date: 2020-10-31-13-28-36 +.. nonce: 6aDbty +.. section: Library + +``binhex.binhex()`` consisently writes macOS 9 line endings. + +.. + +.. bpo: 26789 +.. date: 2020-10-31-01-16-49 +.. nonce: 9BdNAt +.. section: Library + +The :class:`logging.FileHandler` class now keeps a reference to the builtin +:func:`open` function to be able to open or reopen the file during Python +finalization. Fix errors like: ``NameError: name 'open' is not defined``. +Patch by Victor Stinner. + +.. + +.. bpo: 42157 +.. date: 2020-10-26-23-29-16 +.. nonce: 4wuwTe +.. section: Library + +Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal +PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was +moved to the internal C API. Patch by Victor Stinner. + +.. + +.. bpo: 42157 +.. date: 2020-10-26-19-08-07 +.. nonce: Bdpa04 +.. section: Library + +Convert the :mod:`unicodedata` extension module to the multiphase +initialization API (:pep:`489`) and convert the ``unicodedata.UCD`` static +type to a heap type. Patch by Mohamed Koubaa and Victor Stinner. + +.. + +.. bpo: 42146 +.. date: 2020-10-25-19-25-02 +.. nonce: 6A8uvS +.. section: Library + +Fix memory leak in :func:`subprocess.Popen` in case an uid (gid) specified +in `user` (`group`, `extra_groups`) overflows `uid_t` (`gid_t`). + +.. + +.. bpo: 42103 +.. date: 2020-10-23-19-20-14 +.. nonce: C5obK2 +.. section: Library + +:exc:`~plistlib.InvalidFileException` and :exc:`RecursionError` are now the +only errors caused by loading malformed binary Plist file (previously +ValueError and TypeError could be raised in some specific cases). + +.. + +.. bpo: 41490 +.. date: 2020-10-23-08-54-47 +.. nonce: -Yk6OD +.. section: Library + +In ``importlib.resources``, ``.path`` method is more aggressive about +releasing handles to zipfile objects early, enabling use-cases like certifi +to leave the context open but delete the underlying zip file. + +.. + +.. bpo: 41052 +.. date: 2020-10-21-23-45-02 +.. nonce: 3N7J2J +.. section: Library + +Pickling heap types implemented in C with protocols 0 and 1 raises now an +error instead of producing incorrect data. + +.. + +.. bpo: 42089 +.. date: 2020-10-19-16-53-19 +.. nonce: R1dthW +.. section: Library + +In ``importlib.metadata.PackageNotFoundError``, make reference to the +package metadata being missing to improve the user experience. + +.. + +.. bpo: 41491 +.. date: 2020-10-19-14-02-09 +.. nonce: d1BUWH +.. section: Library + +plistlib: fix parsing XML plists with hexadecimal integer values + +.. + +.. bpo: 42065 +.. date: 2020-10-17-23-17-18 +.. nonce: 85BsRA +.. section: Library + +Fix an incorrectly formatted error from :meth:`_codecs.charmap_decode` when +called with a mapped value outside the range of valid Unicode code points. +PR by Max Bernstein. + +.. + +.. bpo: 41966 +.. date: 2020-10-17-07-52-53 +.. nonce: gwEQRZ +.. section: Library + +Fix pickling pure Python :class:`datetime.time` subclasses. Patch by Dean +Inwood. + +.. + +.. bpo: 19270 +.. date: 2020-10-16-22-48-01 +.. nonce: jd_gkA +.. section: Library + +:meth:`sched.scheduler.cancel()` will now cancel the correct event, if two +events with same priority are scheduled for the same time. Patch by Bar +Harel. + +.. + +.. bpo: 28660 +.. date: 2020-10-16-16-08-04 +.. nonce: eX9pvD +.. section: Library + +:func:`textwrap.wrap` now attempts to break long words after hyphens when +``break_long_words=True`` and ``break_on_hyphens=True``. + +.. + +.. bpo: 35823 +.. date: 2020-10-16-07-45-35 +.. nonce: SNQo56 +.. section: Library + +Use ``vfork()`` instead of ``fork()`` for :func:`subprocess.Popen` on Linux +to improve performance in cases where it is deemed safe. + +.. + +.. bpo: 42043 +.. date: 2020-10-15-17-20-37 +.. nonce: OS0p_v +.. section: Library + +Add support for ``zipfile.Path`` inheritance. ``zipfile.Path.is_file()`` now +returns False for non-existent names. ``zipfile.Path`` objects now expose a +``.filename`` attribute and rely on that to resolve ``.name`` and +``.parent`` when the ``Path`` object is at the root of the zipfile. + +.. + +.. bpo: 42021 +.. date: 2020-10-12-21-21-24 +.. nonce: 8yv_8- +.. section: Library + +Fix possible ref leaks in :mod:`sqlite3` module init. + +.. + +.. bpo: 39101 +.. date: 2020-10-11-21-43-03 +.. nonce: -I49Pm +.. section: Library + +Fixed tests using IsolatedAsyncioTestCase from hanging on BaseExceptions. + +.. + +.. bpo: 41976 +.. date: 2020-10-08-18-22-28 +.. nonce: Svm0wb +.. section: Library + +Fixed a bug that was causing :func:`ctypes.util.find_library` to return +``None`` when triying to locate a library in an environment when gcc>=9 is +available and ``ldconfig`` is not. Patch by Pablo Galindo + +.. + +.. bpo: 41943 +.. date: 2020-10-07-18-36-03 +.. nonce: Pt55fT +.. section: Library + +Fix bug where TestCase.assertLogs doesn't correctly filter messages by +level. + +.. + +.. bpo: 41923 +.. date: 2020-10-03-23-14-50 +.. nonce: Buonw9 +.. section: Library + +Implement :pep:`613`, introducing :data:`typing.TypeAlias` annotation. + +.. + +.. bpo: 41905 +.. date: 2020-10-01-21-11-03 +.. nonce: _JpjR4 +.. section: Library + +A new function in abc: *update_abstractmethods* to re-calculate an abstract +class's abstract status. In addition, *dataclass* has been changed to call +this function. + +.. + +.. bpo: 23706 +.. date: 2020-09-30-11-05-11 +.. nonce: dHTGjF +.. section: Library + +Added *newline* parameter to ``pathlib.Path.write_text()``. + +.. + +.. bpo: 41876 +.. date: 2020-09-29-16-23-54 +.. nonce: QicdDU +.. section: Library + +Tkinter font class repr uses font name + +.. + +.. bpo: 41831 +.. date: 2020-09-22-11-07-50 +.. nonce: k-Eop_ +.. section: Library + +``str()`` for the ``type`` attribute of the ``tkinter.Event`` object always +returns now the numeric code returned by Tk instead of the name of the event +type. + +.. + +.. bpo: 39337 +.. date: 2020-09-13-02-02-18 +.. nonce: L3NXTt +.. section: Library + +:func:`encodings.normalize_encoding` now ignores non-ASCII characters. + +.. + +.. bpo: 41747 +.. date: 2020-09-08-23-41-29 +.. nonce: M6wLKv +.. section: Library + +Ensure all methods that generated from :func:`dataclasses.dataclass` objects +now have the proper ``__qualname__`` attribute referring to the class they +belong to. Patch by Batuhan Taskaya. + +.. + +.. bpo: 30681 +.. date: 2020-09-04-17-33-04 +.. nonce: LR4fnY +.. section: Library + +Handle exceptions caused by unparseable date headers when using email +"default" policy. Patch by Tim Bell, Georges Toth + +.. + +.. bpo: 41586 +.. date: 2020-08-19-08-32-13 +.. nonce: IYjmjK +.. section: Library + +Add F_SETPIPE_SZ and F_GETPIPE_SZ to fcntl module. Allow setting pipesize on +subprocess.Popen. + +.. + +.. bpo: 41229 +.. date: 2020-07-19-20-10-41 +.. nonce: p8rJa2 +.. section: Library + +Add ``contextlib.aclosing`` for deterministic cleanup of async generators +which is analogous to ``contextlib.closing`` for non-async generators. Patch +by Joongi Kim and John Belmonte. + +.. + +.. bpo: 16936 +.. date: 2020-07-08-09-45-00 +.. nonce: z8o8Pn +.. section: Library + +Allow ``ctypes.wintypes`` to be imported on non-Windows systems. + +.. + +.. bpo: 4356 +.. date: 2020-05-31-10-48-47 +.. nonce: P8kXqp +.. section: Library + +Add a key function to the bisect module. + +.. + +.. bpo: 40592 +.. date: 2020-05-14-16-01-34 +.. nonce: Cmk855 +.. section: Library + +:func:`shutil.which` now ignores empty entries in :envvar:`PATHEXT` instead +of treating them as a match. + +.. + +.. bpo: 40492 +.. date: 2020-05-04-12-16-00 +.. nonce: ONk9Na +.. section: Library + +Fix ``--outfile`` for :mod:`cProfile` / :mod:`profile` not writing the +output file in the original directory when the program being profiled +changes the working directory. PR by Anthony Sottile. + +.. + +.. bpo: 34204 +.. date: 2020-04-21-17-18-33 +.. nonce: 9wXTtY +.. section: Library + +The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default +instead of :mod:`pickle` protocol ``3``. + +.. + +.. bpo: 27321 +.. date: 2020-01-19-18-40-26 +.. nonce: 8e6SpM +.. section: Library + +Fixed KeyError exception when flattening an email to a string attempts to +replace a non-existent Content-Transfer-Encoding header. + +.. + +.. bpo: 38976 +.. date: 2019-12-05-05-22-49 +.. nonce: 5MG7Uu +.. section: Library + +The :mod:`http.cookiejar` module now supports the parsing of cookies in +CURL-style cookiejar files through MozillaCookieJar on all platforms. +Previously, such cookie entries would be silently ignored when loading a +cookiejar with such entries. + +Additionally, the HTTP Only attribute is persisted in the object, and will +be correctly written to file if the MozillaCookieJar object is subsequently +dumped. + +.. + +.. bpo: 42061 +.. date: 2020-10-28-21-39-45 +.. nonce: _x-0sg +.. section: Documentation + +Document __format__ functionality for IP addresses. + +.. + +.. bpo: 42010 +.. date: 2020-10-21-02-21-14 +.. nonce: 76vJ0u +.. section: Documentation + +Clarify that subscription expressions are also valid for certain +:term:`classes ` and :term:`types ` in the standard library, +and for user-defined classes and types if the classmethod +:meth:`__class_getitem__` is provided. + +.. + +.. bpo: 41805 +.. date: 2020-10-10-01-36-37 +.. nonce: l-CGv5 +.. section: Documentation + +Documented :ref:`generic alias type ` and +:data:`types.GenericAlias`. Also added an entry in glossary for +:term:`generic types `. + +.. + +.. bpo: 39693 +.. date: 2020-02-24-09-02-05 +.. nonce: QXw0Fm +.. section: Documentation + +Fix tarfile's extractfile documentation + +.. + +.. bpo: 39416 +.. date: 2020-01-22-05-14-53 +.. nonce: uYjhEm +.. section: Documentation + +Document some restrictions on the default string representations of numeric +classes. + +.. + +.. bpo: 41739 +.. date: 2020-10-12-00-11-47 +.. nonce: wSCc4K +.. section: Tests + +Fix test_logging.test_race_between_set_target_and_flush(): the test now +waits until all threads complete to avoid leaking running threads. + +.. + +.. bpo: 41970 +.. date: 2020-10-08-14-00-17 +.. nonce: aZ8QFf +.. section: Tests + +Avoid a test failure in ``test_lib2to3`` if the module has already imported +at the time the test executes. Patch by Pablo Galindo. + +.. + +.. bpo: 41944 +.. date: 2020-10-05-17-43-46 +.. nonce: rf1dYb +.. section: Tests + +Tests for CJK codecs no longer call ``eval()`` on content received via HTTP. + +.. + +.. bpo: 41306 +.. date: 2020-08-03-13-44-37 +.. nonce: VDoWXI +.. section: Tests + +Fixed a failure in ``test_tk.test_widgets.ScaleTest`` happening when +executing the test with Tk 8.6.10. + +.. + +.. bpo: 38980 +.. date: 2020-10-21-18-31-54 +.. nonce: xz7BNd +.. section: Build + +Add ``-fno-semantic-interposition`` to both the compile and link line when +building with ``--enable-optimizations``. Patch by Victor Stinner and Pablo +Galindo. + +.. + +.. bpo: 38439 +.. date: 2020-10-20-13-19-42 +.. nonce: eMLi-t +.. section: Windows + +Updates the icons for IDLE in the Windows Store package. + +.. + +.. bpo: 38252 +.. date: 2020-10-18-18-43-45 +.. nonce: 7Nlepg +.. section: Windows + +Use 8-byte step to detect ASCII sequence in 64-bit Windows build. + +.. + +.. bpo: 39107 +.. date: 2020-09-24-23-09-40 +.. nonce: GbUZvD +.. section: Windows + +Update Tcl and Tk to 8.6.10 in Windows installer. + +.. + +.. bpo: 41557 +.. date: 2020-08-26-09-35-06 +.. nonce: vt00cQ +.. section: Windows + +Update Windows installer to use SQLite 3.33.0. + +.. + +.. bpo: 38324 +.. date: 2020-05-30-02-46-43 +.. nonce: 476M-5 +.. section: Windows + +Avoid Unicode errors when accessing certain locale data on Windows. + +.. + +.. bpo: 41471 +.. date: 2020-10-19-12-25-19 +.. nonce: gwA7un +.. section: macOS + +Ignore invalid prefix lengths in system proxy excludes. + +.. + +.. bpo: 33987 +.. date: 2020-10-24-21-27-37 +.. nonce: fIh9JL +.. section: IDLE + +Mostly finish using ttk widgets, mainly for editor, settings, and searches. +Some patches by Mark Roseman. + +.. + +.. bpo: 40511 +.. date: 2020-06-16-12-16-13 +.. nonce: XkihpM +.. section: IDLE + +Typing opening and closing parentheses inside the parentheses of a function +call will no longer cause unnecessary "flashing" off and on of an existing +open call-tip, e.g. when typed in a string literal. + +.. + +.. bpo: 38439 +.. date: 2020-04-22-09-37-40 +.. nonce: ieXL-c +.. section: IDLE + +Add a 256×256 pixel IDLE icon to the Windows .ico file. Created by Andrew +Clover. Remove the low-color gif variations from the .ico file. + +.. + +.. bpo: 42157 +.. date: 2020-10-16-10-47-17 +.. nonce: e3BcPM +.. section: C API + +The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API +``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. Patch by +Victor Stinner. + +.. + +.. bpo: 42015 +.. date: 2020-10-12-20-13-58 +.. nonce: X4H2_V +.. section: C API + +Fix potential crash in deallocating method objects when dynamically +allocated `PyMethodDef`'s lifetime is managed through the ``self`` argument +of a `PyCFunction`. + +.. + +.. bpo: 40423 +.. date: 2020-10-11-19-17-44 +.. nonce: GsmgEj +.. section: C API + +The :mod:`subprocess` module and ``os.closerange`` will now use the +``close_range(low, high, flags)`` syscall when it is available for more +efficient closing of ranges of descriptors. + +.. + +.. bpo: 41845 +.. date: 2020-10-11-05-05-53 +.. nonce: ZFvuQM +.. section: C API + +:c:func:`PyObject_GenericGetDict` is available again in the limited API when +targeting 3.10 or later. + +.. + +.. bpo: 40422 +.. date: 2020-10-10-14-05-24 +.. nonce: sh8IDY +.. section: C API + +Add `_Py_closerange` function to provide performant closing of a range of +file descriptors. + +.. + +.. bpo: 41986 +.. date: 2020-10-09-22-50-46 +.. nonce: JUPE59 +.. section: C API + +:c:data:`Py_FileSystemDefaultEncodeErrors` and :c:data:`Py_UTF8Mode` are +available again in limited API. + +.. + +.. bpo: 41756 +.. date: 2020-09-28-14-31-07 +.. nonce: ZZ5wJG +.. section: C API + +Add `PyIter_Send` function to allow sending value into +generator/coroutine/iterator without raising StopIteration exception to +signal return. + +.. + +.. bpo: 41784 +.. date: 2020-09-14-10-17-00 +.. nonce: Yl4gI2 +.. section: C API + +Added ``PyUnicode_AsUTF8AndSize`` to the limited C API. diff --git a/Misc/NEWS.d/next/Build/2020-10-21-18-31-54.bpo-38980.xz7BNd.rst b/Misc/NEWS.d/next/Build/2020-10-21-18-31-54.bpo-38980.xz7BNd.rst deleted file mode 100644 index c9d34cf12c2..00000000000 --- a/Misc/NEWS.d/next/Build/2020-10-21-18-31-54.bpo-38980.xz7BNd.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add ``-fno-semantic-interposition`` to both the compile and link line when -building with ``--enable-optimizations``. Patch by Victor Stinner and Pablo -Galindo. diff --git a/Misc/NEWS.d/next/C API/2020-09-14-10-17-00.bpo-41784.Yl4gI2.rst b/Misc/NEWS.d/next/C API/2020-09-14-10-17-00.bpo-41784.Yl4gI2.rst deleted file mode 100644 index f09e0879ad1..00000000000 --- a/Misc/NEWS.d/next/C API/2020-09-14-10-17-00.bpo-41784.Yl4gI2.rst +++ /dev/null @@ -1 +0,0 @@ -Added ``PyUnicode_AsUTF8AndSize`` to the limited C API. diff --git a/Misc/NEWS.d/next/C API/2020-09-28-14-31-07.bpo-41756.ZZ5wJG.rst b/Misc/NEWS.d/next/C API/2020-09-28-14-31-07.bpo-41756.ZZ5wJG.rst deleted file mode 100644 index f7e27b44015..00000000000 --- a/Misc/NEWS.d/next/C API/2020-09-28-14-31-07.bpo-41756.ZZ5wJG.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add `PyIter_Send` function to allow sending value into -generator/coroutine/iterator without raising StopIteration exception to -signal return. diff --git a/Misc/NEWS.d/next/C API/2020-10-09-22-50-46.bpo-41986.JUPE59.rst b/Misc/NEWS.d/next/C API/2020-10-09-22-50-46.bpo-41986.JUPE59.rst deleted file mode 100644 index d456ba66baf..00000000000 --- a/Misc/NEWS.d/next/C API/2020-10-09-22-50-46.bpo-41986.JUPE59.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:data:`Py_FileSystemDefaultEncodeErrors` and :c:data:`Py_UTF8Mode` are -available again in limited API. diff --git a/Misc/NEWS.d/next/C API/2020-10-10-14-05-24.bpo-40422.sh8IDY.rst b/Misc/NEWS.d/next/C API/2020-10-10-14-05-24.bpo-40422.sh8IDY.rst deleted file mode 100644 index 1b6d9e034b5..00000000000 --- a/Misc/NEWS.d/next/C API/2020-10-10-14-05-24.bpo-40422.sh8IDY.rst +++ /dev/null @@ -1 +0,0 @@ -Add `_Py_closerange` function to provide performant closing of a range of file descriptors. \ No newline at end of file diff --git a/Misc/NEWS.d/next/C API/2020-10-11-05-05-53.bpo-41845.ZFvuQM.rst b/Misc/NEWS.d/next/C API/2020-10-11-05-05-53.bpo-41845.ZFvuQM.rst deleted file mode 100644 index 31d3154c3c6..00000000000 --- a/Misc/NEWS.d/next/C API/2020-10-11-05-05-53.bpo-41845.ZFvuQM.rst +++ /dev/null @@ -1,2 +0,0 @@ -:c:func:`PyObject_GenericGetDict` is available again in the limited API -when targeting 3.10 or later. diff --git a/Misc/NEWS.d/next/C API/2020-10-11-19-17-44.bpo-40423.GsmgEj.rst b/Misc/NEWS.d/next/C API/2020-10-11-19-17-44.bpo-40423.GsmgEj.rst deleted file mode 100644 index 44e571ebf86..00000000000 --- a/Misc/NEWS.d/next/C API/2020-10-11-19-17-44.bpo-40423.GsmgEj.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`subprocess` module and ``os.closerange`` will now use the -``close_range(low, high, flags)`` syscall when it is available for more -efficient closing of ranges of descriptors. \ No newline at end of file diff --git a/Misc/NEWS.d/next/C API/2020-10-12-20-13-58.bpo-42015.X4H2_V.rst b/Misc/NEWS.d/next/C API/2020-10-12-20-13-58.bpo-42015.X4H2_V.rst deleted file mode 100644 index d77619f64bb..00000000000 --- a/Misc/NEWS.d/next/C API/2020-10-12-20-13-58.bpo-42015.X4H2_V.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix potential crash in deallocating method objects when dynamically -allocated `PyMethodDef`'s lifetime is managed through the ``self`` -argument of a `PyCFunction`. diff --git a/Misc/NEWS.d/next/C API/2020-10-16-10-47-17.bpo-42157.e3BcPM.rst b/Misc/NEWS.d/next/C API/2020-10-16-10-47-17.bpo-42157.e3BcPM.rst deleted file mode 100644 index 65a56188fa0..00000000000 --- a/Misc/NEWS.d/next/C API/2020-10-16-10-47-17.bpo-42157.e3BcPM.rst +++ /dev/null @@ -1,3 +0,0 @@ -The private ``_PyUnicode_Name_CAPI`` structure of the PyCapsule API -``unicodedata.ucnhash_CAPI`` has been moved to the internal C API. -Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-05-27-16-08-16.bpo-38605.rcs2uK.rst b/Misc/NEWS.d/next/Core and Builtins/2020-05-27-16-08-16.bpo-38605.rcs2uK.rst deleted file mode 100644 index cbfe6e23523..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-05-27-16-08-16.bpo-38605.rcs2uK.rst +++ /dev/null @@ -1,3 +0,0 @@ -Enable ``from __future__ import annotations`` (:pep:`563`) by default. -The values found in :attr:`__annotations__` dicts are now strings, e.g. -``{"x": "int"}`` instead of ``{"x": int}``. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst b/Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst deleted file mode 100644 index d2978f9b4ec..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-08-07-13-42-48.bpo-41435.qPWjJA.rst +++ /dev/null @@ -1 +0,0 @@ -Add `sys._current_exceptions()` function to retrieve a dictionary mapping each thread's identifier to the topmost exception currently active in that thread at the time the function is called. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-01-22-44-23.bpo-41902.iLoMVF.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-01-22-44-23.bpo-41902.iLoMVF.rst deleted file mode 100644 index 738ef5aec95..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-01-22-44-23.bpo-41902.iLoMVF.rst +++ /dev/null @@ -1 +0,0 @@ -Micro optimization for range.index if step is 1. Patch by Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-02-11-35-33.bpo-41894.ffmtOt.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-02-11-35-33.bpo-41894.ffmtOt.rst deleted file mode 100644 index 571f5dae1a4..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-02-11-35-33.bpo-41894.ffmtOt.rst +++ /dev/null @@ -1,3 +0,0 @@ -When loading a native module and a load failure occurs, prevent a possible -UnicodeDecodeError when not running in a UTF-8 locale by decoding the load -error message using the current locale's encoding. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-02-13-32-05.bpo-41902.ZKTxzW.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-02-13-32-05.bpo-41902.ZKTxzW.rst deleted file mode 100644 index b118a6a36fa..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-02-13-32-05.bpo-41902.ZKTxzW.rst +++ /dev/null @@ -1,3 +0,0 @@ -Micro optimization when compute :c:member:`~PySequenceMethods.sq_item` and -:c:member:`~PyMappingMethods.mp_subscript` of :class:`range`. Patch by -Dong-hee Na. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-08-09-58-19.bpo-41974.8B-q8O.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-08-09-58-19.bpo-41974.8B-q8O.rst deleted file mode 100644 index 034cfede84b..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-08-09-58-19.bpo-41974.8B-q8O.rst +++ /dev/null @@ -1,4 +0,0 @@ -Removed special methods ``__int__``, ``__float__``, ``__floordiv__``, -``__mod__``, ``__divmod__``, ``__rfloordiv__``, ``__rmod__`` and -``__rdivmod__`` of the :class:`complex` class. They always raised -a :exc:`TypeError`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-09-10-55-50.bpo-41979.ImXIk2.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-09-10-55-50.bpo-41979.ImXIk2.rst deleted file mode 100644 index 3250309ca22..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-09-10-55-50.bpo-41979.ImXIk2.rst +++ /dev/null @@ -1 +0,0 @@ -Star-unpacking is now allowed for with item's targets in the PEG parser. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-10-13-53-52.bpo-41993.YMzixQ.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-10-13-53-52.bpo-41993.YMzixQ.rst deleted file mode 100644 index 3669cf11ea4..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-10-13-53-52.bpo-41993.YMzixQ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed potential issues with removing not completely initialized module from -``sys.modules`` when import fails. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-14-16-19-43.bpo-41984.SEtKMr.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-14-16-19-43.bpo-41984.SEtKMr.rst deleted file mode 100644 index e70d5dc2b8d..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-14-16-19-43.bpo-41984.SEtKMr.rst +++ /dev/null @@ -1,2 +0,0 @@ -The garbage collector now tracks all user-defined classes. Patch by Brandt -Bucher. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-15-21-55-32.bpo-42030.PmU2CA.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-15-21-55-32.bpo-42030.PmU2CA.rst deleted file mode 100644 index e8c691d8096..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-15-21-55-32.bpo-42030.PmU2CA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Support for the legacy AIX-specific shared library loading support has been -removed. All versions of AIX since 4.3 have supported and defaulted to using -the common Unix mechanism instead. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-20-04-24-07.bpo-42093.ooZZNh.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-20-04-24-07.bpo-42093.ooZZNh.rst deleted file mode 100644 index 36a12c1c1cb..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-20-04-24-07.bpo-42093.ooZZNh.rst +++ /dev/null @@ -1,2 +0,0 @@ -The ``LOAD_ATTR`` instruction now uses new "per opcode cache" mechanism and -it is about 36% faster now. Patch by Pablo Galindo and Yury Selivanov. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-21-14-40-54.bpo-41910.CzBMit.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-21-14-40-54.bpo-41910.CzBMit.rst deleted file mode 100644 index a40e2519a62..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-21-14-40-54.bpo-41910.CzBMit.rst +++ /dev/null @@ -1 +0,0 @@ -Document the default implementation of `object.__eq__`. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-23-02-43-24.bpo-42123.64gJWC.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-23-02-43-24.bpo-42123.64gJWC.rst deleted file mode 100644 index 6461efd76f0..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-23-02-43-24.bpo-42123.64gJWC.rst +++ /dev/null @@ -1,3 +0,0 @@ -Run the parser two times. On the first run, disable all the rules that only -generate better error messages to gain performance. If there's a parse -failure, run the parser a second time with those enabled. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-25-21-14-18.bpo-42150.b70u_T.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-25-21-14-18.bpo-42150.b70u_T.rst deleted file mode 100644 index 62fabb857aa..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-25-21-14-18.bpo-42150.b70u_T.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix possible buffer overflow in the new parser when checking for -continuation lines. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-27-18-32-49.bpo-41659.d4a-8o.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-27-18-32-49.bpo-41659.d4a-8o.rst deleted file mode 100644 index 038749a7b16..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-27-18-32-49.bpo-41659.d4a-8o.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix a bug in the parser, where a curly brace following a `primary` didn't fail immediately. -This led to invalid expressions like `a {b}` to throw a :exc:`SyntaxError` with a wrong offset, -or invalid expressions ending with a curly brace like `a {` to not fail immediately in the REPL. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-27-21-34-05.bpo-42143.N6KXUO.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-27-21-34-05.bpo-42143.N6KXUO.rst deleted file mode 100644 index 2b16e69da73..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-27-21-34-05.bpo-42143.N6KXUO.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix handling of errors during creation of ``PyFunctionObject``, which resulted -in operations on uninitialized memory. Patch by Yonatan Goldschmidt. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst deleted file mode 100644 index b3ac08edd84..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-29-12-49-08.bpo-41796.tkGdHq.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`ast` module internal state is now per interpreter. Patch by Victor -Stinner. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-30-13-11-01.bpo-42206.xxssR8.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-30-13-11-01.bpo-42206.xxssR8.rst deleted file mode 100644 index b9eb135fec5..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-30-13-11-01.bpo-42206.xxssR8.rst +++ /dev/null @@ -1,2 +0,0 @@ -Propagate and raise the errors caused by :c:func:`PyAST_Validate` in the -parser. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-30-22-16-30.bpo-42214.lXskM_.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-30-22-16-30.bpo-42214.lXskM_.rst deleted file mode 100644 index 3f85bbe8390..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-30-22-16-30.bpo-42214.lXskM_.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed a possible crash in the PEG parser when checking for the '!=' token in -the ``barry_as_flufl`` rule. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-10-31-17-50-23.bpo-42218.Dp_Z3v.rst b/Misc/NEWS.d/next/Core and Builtins/2020-10-31-17-50-23.bpo-42218.Dp_Z3v.rst deleted file mode 100644 index a38a310e4b4..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-10-31-17-50-23.bpo-42218.Dp_Z3v.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug in the PEG parser that was causing crashes in debug mode. Now errors are checked -in left-recursive rules to avoid cases where such errors do not get handled in time and appear -as long-distance crashes in other places. diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-01-21-21-38.bpo-42236.MPx-NK.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-01-21-21-38.bpo-42236.MPx-NK.rst deleted file mode 100644 index 22e8c534ff8..00000000000 --- a/Misc/NEWS.d/next/Core and Builtins/2020-11-01-21-21-38.bpo-42236.MPx-NK.rst +++ /dev/null @@ -1,2 +0,0 @@ -If the ``nl_langinfo(CODESET)`` function returns an empty string, Python now -uses UTF-8 as the filesystem encoding. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Documentation/2020-01-22-05-14-53.bpo-39416.uYjhEm.rst b/Misc/NEWS.d/next/Documentation/2020-01-22-05-14-53.bpo-39416.uYjhEm.rst deleted file mode 100644 index 279a5f18ff8..00000000000 --- a/Misc/NEWS.d/next/Documentation/2020-01-22-05-14-53.bpo-39416.uYjhEm.rst +++ /dev/null @@ -1 +0,0 @@ -Document some restrictions on the default string representations of numeric classes. diff --git a/Misc/NEWS.d/next/Documentation/2020-02-24-09-02-05.bpo-39693.QXw0Fm.rst b/Misc/NEWS.d/next/Documentation/2020-02-24-09-02-05.bpo-39693.QXw0Fm.rst deleted file mode 100644 index 86049c53681..00000000000 --- a/Misc/NEWS.d/next/Documentation/2020-02-24-09-02-05.bpo-39693.QXw0Fm.rst +++ /dev/null @@ -1 +0,0 @@ -Fix tarfile's extractfile documentation diff --git a/Misc/NEWS.d/next/Documentation/2020-10-10-01-36-37.bpo-41805.l-CGv5.rst b/Misc/NEWS.d/next/Documentation/2020-10-10-01-36-37.bpo-41805.l-CGv5.rst deleted file mode 100644 index 9c9134350a3..00000000000 --- a/Misc/NEWS.d/next/Documentation/2020-10-10-01-36-37.bpo-41805.l-CGv5.rst +++ /dev/null @@ -1,3 +0,0 @@ -Documented :ref:`generic alias type ` and -:data:`types.GenericAlias`. Also added an entry in glossary for -:term:`generic types `. diff --git a/Misc/NEWS.d/next/Documentation/2020-10-21-02-21-14.bpo-42010.76vJ0u.rst b/Misc/NEWS.d/next/Documentation/2020-10-21-02-21-14.bpo-42010.76vJ0u.rst deleted file mode 100644 index 2a0cbf10757..00000000000 --- a/Misc/NEWS.d/next/Documentation/2020-10-21-02-21-14.bpo-42010.76vJ0u.rst +++ /dev/null @@ -1,4 +0,0 @@ -Clarify that subscription expressions are also valid for certain -:term:`classes ` and :term:`types ` in the standard library, and -for user-defined classes and types if the classmethod -:meth:`__class_getitem__` is provided. diff --git a/Misc/NEWS.d/next/Documentation/2020-10-28-21-39-45.bpo-42061._x-0sg.rst b/Misc/NEWS.d/next/Documentation/2020-10-28-21-39-45.bpo-42061._x-0sg.rst deleted file mode 100644 index b38bb843501..00000000000 --- a/Misc/NEWS.d/next/Documentation/2020-10-28-21-39-45.bpo-42061._x-0sg.rst +++ /dev/null @@ -1 +0,0 @@ -Document __format__ functionality for IP addresses. \ No newline at end of file diff --git a/Misc/NEWS.d/next/IDLE/2020-04-22-09-37-40.bpo-38439.ieXL-c.rst b/Misc/NEWS.d/next/IDLE/2020-04-22-09-37-40.bpo-38439.ieXL-c.rst deleted file mode 100644 index d8d59015f20..00000000000 --- a/Misc/NEWS.d/next/IDLE/2020-04-22-09-37-40.bpo-38439.ieXL-c.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add a 256×256 pixel IDLE icon to the Windows .ico file. Created by Andrew -Clover. Remove the low-color gif variations from the .ico file. diff --git a/Misc/NEWS.d/next/IDLE/2020-06-16-12-16-13.bpo-40511.XkihpM.rst b/Misc/NEWS.d/next/IDLE/2020-06-16-12-16-13.bpo-40511.XkihpM.rst deleted file mode 100644 index cc967981381..00000000000 --- a/Misc/NEWS.d/next/IDLE/2020-06-16-12-16-13.bpo-40511.XkihpM.rst +++ /dev/null @@ -1,3 +0,0 @@ -Typing opening and closing parentheses inside the parentheses of a function -call will no longer cause unnecessary "flashing" off and on of an existing -open call-tip, e.g. when typed in a string literal. diff --git a/Misc/NEWS.d/next/IDLE/2020-10-24-21-27-37.bpo-33987.fIh9JL.rst b/Misc/NEWS.d/next/IDLE/2020-10-24-21-27-37.bpo-33987.fIh9JL.rst deleted file mode 100644 index 1e67edc03c6..00000000000 --- a/Misc/NEWS.d/next/IDLE/2020-10-24-21-27-37.bpo-33987.fIh9JL.rst +++ /dev/null @@ -1,3 +0,0 @@ -Mostly finish using ttk widgets, mainly for editor, settings, -and searches. Some patches by Mark Roseman. - diff --git a/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst b/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst deleted file mode 100644 index 7a48943a6c6..00000000000 --- a/Misc/NEWS.d/next/Library/2019-12-05-05-22-49.bpo-38976.5MG7Uu.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :mod:`http.cookiejar` module now supports the parsing of cookies in CURL-style cookiejar files through MozillaCookieJar -on all platforms. Previously, such cookie entries would be silently ignored when loading a cookiejar with such entries. - -Additionally, the HTTP Only attribute is persisted in the object, and will be correctly written to file if the MozillaCookieJar object is subsequently dumped. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-01-19-18-40-26.bpo-27321.8e6SpM.rst b/Misc/NEWS.d/next/Library/2020-01-19-18-40-26.bpo-27321.8e6SpM.rst deleted file mode 100644 index 28acf7f6ef9..00000000000 --- a/Misc/NEWS.d/next/Library/2020-01-19-18-40-26.bpo-27321.8e6SpM.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fixed KeyError exception when flattening an email to a string attempts to -replace a non-existent Content-Transfer-Encoding header. diff --git a/Misc/NEWS.d/next/Library/2020-04-21-17-18-33.bpo-34204.9wXTtY.rst b/Misc/NEWS.d/next/Library/2020-04-21-17-18-33.bpo-34204.9wXTtY.rst deleted file mode 100644 index bce6d39148a..00000000000 --- a/Misc/NEWS.d/next/Library/2020-04-21-17-18-33.bpo-34204.9wXTtY.rst +++ /dev/null @@ -1,2 +0,0 @@ -The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default -instead of :mod:`pickle` protocol ``3``. diff --git a/Misc/NEWS.d/next/Library/2020-05-04-12-16-00.bpo-40492.ONk9Na.rst b/Misc/NEWS.d/next/Library/2020-05-04-12-16-00.bpo-40492.ONk9Na.rst deleted file mode 100644 index 86bc08c79e2..00000000000 --- a/Misc/NEWS.d/next/Library/2020-05-04-12-16-00.bpo-40492.ONk9Na.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix ``--outfile`` for :mod:`cProfile` / :mod:`profile` not writing the output -file in the original directory when the program being profiled changes the -working directory. PR by Anthony Sottile. diff --git a/Misc/NEWS.d/next/Library/2020-05-14-16-01-34.bpo-40592.Cmk855.rst b/Misc/NEWS.d/next/Library/2020-05-14-16-01-34.bpo-40592.Cmk855.rst deleted file mode 100644 index 3211a1bc345..00000000000 --- a/Misc/NEWS.d/next/Library/2020-05-14-16-01-34.bpo-40592.Cmk855.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`shutil.which` now ignores empty entries in :envvar:`PATHEXT` instead of treating them as a match. diff --git a/Misc/NEWS.d/next/Library/2020-05-31-10-48-47.bpo-4356.P8kXqp.rst b/Misc/NEWS.d/next/Library/2020-05-31-10-48-47.bpo-4356.P8kXqp.rst deleted file mode 100644 index f5211d8a76f..00000000000 --- a/Misc/NEWS.d/next/Library/2020-05-31-10-48-47.bpo-4356.P8kXqp.rst +++ /dev/null @@ -1 +0,0 @@ -Add a key function to the bisect module. diff --git a/Misc/NEWS.d/next/Library/2020-07-08-09-45-00.bpo-16936.z8o8Pn.rst b/Misc/NEWS.d/next/Library/2020-07-08-09-45-00.bpo-16936.z8o8Pn.rst deleted file mode 100644 index c76db4eedec..00000000000 --- a/Misc/NEWS.d/next/Library/2020-07-08-09-45-00.bpo-16936.z8o8Pn.rst +++ /dev/null @@ -1 +0,0 @@ -Allow ``ctypes.wintypes`` to be imported on non-Windows systems. diff --git a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst b/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst deleted file mode 100644 index 926133221d4..00000000000 --- a/Misc/NEWS.d/next/Library/2020-07-19-20-10-41.bpo-41229.p8rJa2.rst +++ /dev/null @@ -1,3 +0,0 @@ -Add ``contextlib.aclosing`` for deterministic cleanup of async generators -which is analogous to ``contextlib.closing`` for non-async generators. -Patch by Joongi Kim and John Belmonte. diff --git a/Misc/NEWS.d/next/Library/2020-08-19-08-32-13.bpo-41586.IYjmjK.rst b/Misc/NEWS.d/next/Library/2020-08-19-08-32-13.bpo-41586.IYjmjK.rst deleted file mode 100644 index 40461679ebd..00000000000 --- a/Misc/NEWS.d/next/Library/2020-08-19-08-32-13.bpo-41586.IYjmjK.rst +++ /dev/null @@ -1,2 +0,0 @@ -Add F_SETPIPE_SZ and F_GETPIPE_SZ to fcntl module. Allow setting pipesize on -subprocess.Popen. diff --git a/Misc/NEWS.d/next/Library/2020-09-04-17-33-04.bpo-30681.LR4fnY.rst b/Misc/NEWS.d/next/Library/2020-09-04-17-33-04.bpo-30681.LR4fnY.rst deleted file mode 100644 index 83830e343da..00000000000 --- a/Misc/NEWS.d/next/Library/2020-09-04-17-33-04.bpo-30681.LR4fnY.rst +++ /dev/null @@ -1,2 +0,0 @@ -Handle exceptions caused by unparseable date headers when using email -"default" policy. Patch by Tim Bell, Georges Toth diff --git a/Misc/NEWS.d/next/Library/2020-09-08-23-41-29.bpo-41747.M6wLKv.rst b/Misc/NEWS.d/next/Library/2020-09-08-23-41-29.bpo-41747.M6wLKv.rst deleted file mode 100644 index 0869462f5bf..00000000000 --- a/Misc/NEWS.d/next/Library/2020-09-08-23-41-29.bpo-41747.M6wLKv.rst +++ /dev/null @@ -1,3 +0,0 @@ -Ensure all methods that generated from :func:`dataclasses.dataclass` -objects now have the proper ``__qualname__`` attribute referring to -the class they belong to. Patch by Batuhan Taskaya. diff --git a/Misc/NEWS.d/next/Library/2020-09-13-02-02-18.bpo-39337.L3NXTt.rst b/Misc/NEWS.d/next/Library/2020-09-13-02-02-18.bpo-39337.L3NXTt.rst deleted file mode 100644 index c2b4dbe4d12..00000000000 --- a/Misc/NEWS.d/next/Library/2020-09-13-02-02-18.bpo-39337.L3NXTt.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`encodings.normalize_encoding` now ignores non-ASCII characters. diff --git a/Misc/NEWS.d/next/Library/2020-09-22-11-07-50.bpo-41831.k-Eop_.rst b/Misc/NEWS.d/next/Library/2020-09-22-11-07-50.bpo-41831.k-Eop_.rst deleted file mode 100644 index 84a3f5253a0..00000000000 --- a/Misc/NEWS.d/next/Library/2020-09-22-11-07-50.bpo-41831.k-Eop_.rst +++ /dev/null @@ -1,3 +0,0 @@ -``str()`` for the ``type`` attribute of the ``tkinter.Event`` object always -returns now the numeric code returned by Tk instead of the name of the event -type. diff --git a/Misc/NEWS.d/next/Library/2020-09-29-16-23-54.bpo-41876.QicdDU.rst b/Misc/NEWS.d/next/Library/2020-09-29-16-23-54.bpo-41876.QicdDU.rst deleted file mode 100644 index d4f5f0a37bf..00000000000 --- a/Misc/NEWS.d/next/Library/2020-09-29-16-23-54.bpo-41876.QicdDU.rst +++ /dev/null @@ -1 +0,0 @@ -Tkinter font class repr uses font name \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-09-30-11-05-11.bpo-23706.dHTGjF.rst b/Misc/NEWS.d/next/Library/2020-09-30-11-05-11.bpo-23706.dHTGjF.rst deleted file mode 100644 index b9a69a55e6a..00000000000 --- a/Misc/NEWS.d/next/Library/2020-09-30-11-05-11.bpo-23706.dHTGjF.rst +++ /dev/null @@ -1 +0,0 @@ -Added *newline* parameter to ``pathlib.Path.write_text()``. diff --git a/Misc/NEWS.d/next/Library/2020-10-01-21-11-03.bpo-41905._JpjR4.rst b/Misc/NEWS.d/next/Library/2020-10-01-21-11-03.bpo-41905._JpjR4.rst deleted file mode 100644 index 0d8c0ba6a66..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-01-21-11-03.bpo-41905._JpjR4.rst +++ /dev/null @@ -1 +0,0 @@ -A new function in abc: *update_abstractmethods* to re-calculate an abstract class's abstract status. In addition, *dataclass* has been changed to call this function. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-10-03-23-14-50.bpo-41923.Buonw9.rst b/Misc/NEWS.d/next/Library/2020-10-03-23-14-50.bpo-41923.Buonw9.rst deleted file mode 100644 index dd9a1f709f3..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-03-23-14-50.bpo-41923.Buonw9.rst +++ /dev/null @@ -1 +0,0 @@ -Implement :pep:`613`, introducing :data:`typing.TypeAlias` annotation. diff --git a/Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst b/Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst deleted file mode 100644 index 3a7874d25ae..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-07-18-36-03.bpo-41943.Pt55fT.rst +++ /dev/null @@ -1 +0,0 @@ -Fix bug where TestCase.assertLogs doesn't correctly filter messages by level. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst b/Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst deleted file mode 100644 index c8b3fc77184..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-08-18-22-28.bpo-41976.Svm0wb.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fixed a bug that was causing :func:`ctypes.util.find_library` to return -``None`` when triying to locate a library in an environment when gcc>=9 is -available and ``ldconfig`` is not. Patch by Pablo Galindo diff --git a/Misc/NEWS.d/next/Library/2020-10-11-21-43-03.bpo-39101.-I49Pm.rst b/Misc/NEWS.d/next/Library/2020-10-11-21-43-03.bpo-39101.-I49Pm.rst deleted file mode 100644 index a571e8343cd..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-11-21-43-03.bpo-39101.-I49Pm.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed tests using IsolatedAsyncioTestCase from hanging on BaseExceptions. \ No newline at end of file diff --git a/Misc/NEWS.d/next/Library/2020-10-12-21-21-24.bpo-42021.8yv_8-.rst b/Misc/NEWS.d/next/Library/2020-10-12-21-21-24.bpo-42021.8yv_8-.rst deleted file mode 100644 index 7d71e9a7007..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-12-21-21-24.bpo-42021.8yv_8-.rst +++ /dev/null @@ -1 +0,0 @@ -Fix possible ref leaks in :mod:`sqlite3` module init. diff --git a/Misc/NEWS.d/next/Library/2020-10-15-17-20-37.bpo-42043.OS0p_v.rst b/Misc/NEWS.d/next/Library/2020-10-15-17-20-37.bpo-42043.OS0p_v.rst deleted file mode 100644 index b6b296956c3..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-15-17-20-37.bpo-42043.OS0p_v.rst +++ /dev/null @@ -1,4 +0,0 @@ -Add support for ``zipfile.Path`` inheritance. ``zipfile.Path.is_file()`` now -returns False for non-existent names. ``zipfile.Path`` objects now expose a -``.filename`` attribute and rely on that to resolve ``.name`` and -``.parent`` when the ``Path`` object is at the root of the zipfile. diff --git a/Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst b/Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst deleted file mode 100644 index cd428d36960..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-16-07-45-35.bpo-35823.SNQo56.rst +++ /dev/null @@ -1,2 +0,0 @@ -Use ``vfork()`` instead of ``fork()`` for :func:`subprocess.Popen` on Linux -to improve performance in cases where it is deemed safe. diff --git a/Misc/NEWS.d/next/Library/2020-10-16-16-08-04.bpo-28660.eX9pvD.rst b/Misc/NEWS.d/next/Library/2020-10-16-16-08-04.bpo-28660.eX9pvD.rst deleted file mode 100644 index d67993492f9..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-16-16-08-04.bpo-28660.eX9pvD.rst +++ /dev/null @@ -1 +0,0 @@ -:func:`textwrap.wrap` now attempts to break long words after hyphens when ``break_long_words=True`` and ``break_on_hyphens=True``. diff --git a/Misc/NEWS.d/next/Library/2020-10-16-22-48-01.bpo-19270.jd_gkA.rst b/Misc/NEWS.d/next/Library/2020-10-16-22-48-01.bpo-19270.jd_gkA.rst deleted file mode 100644 index 6330a91a44c..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-16-22-48-01.bpo-19270.jd_gkA.rst +++ /dev/null @@ -1,2 +0,0 @@ -:meth:`sched.scheduler.cancel()` will now cancel the correct event, if two -events with same priority are scheduled for the same time. Patch by Bar Harel. diff --git a/Misc/NEWS.d/next/Library/2020-10-17-07-52-53.bpo-41966.gwEQRZ.rst b/Misc/NEWS.d/next/Library/2020-10-17-07-52-53.bpo-41966.gwEQRZ.rst deleted file mode 100644 index 0e7fad40077..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-17-07-52-53.bpo-41966.gwEQRZ.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix pickling pure Python :class:`datetime.time` subclasses. Patch by Dean -Inwood. diff --git a/Misc/NEWS.d/next/Library/2020-10-17-23-17-18.bpo-42065.85BsRA.rst b/Misc/NEWS.d/next/Library/2020-10-17-23-17-18.bpo-42065.85BsRA.rst deleted file mode 100644 index 83c86c0799e..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-17-23-17-18.bpo-42065.85BsRA.rst +++ /dev/null @@ -1,3 +0,0 @@ -Fix an incorrectly formatted error from :meth:`_codecs.charmap_decode` when -called with a mapped value outside the range of valid Unicode code points. -PR by Max Bernstein. diff --git a/Misc/NEWS.d/next/Library/2020-10-19-14-02-09.bpo-41491.d1BUWH.rst b/Misc/NEWS.d/next/Library/2020-10-19-14-02-09.bpo-41491.d1BUWH.rst deleted file mode 100644 index 4f39c91b284..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-19-14-02-09.bpo-41491.d1BUWH.rst +++ /dev/null @@ -1 +0,0 @@ -plistlib: fix parsing XML plists with hexadecimal integer values diff --git a/Misc/NEWS.d/next/Library/2020-10-19-16-53-19.bpo-42089.R1dthW.rst b/Misc/NEWS.d/next/Library/2020-10-19-16-53-19.bpo-42089.R1dthW.rst deleted file mode 100644 index 3f3affd1a7d..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-19-16-53-19.bpo-42089.R1dthW.rst +++ /dev/null @@ -1,2 +0,0 @@ -In ``importlib.metadata.PackageNotFoundError``, make reference to the -package metadata being missing to improve the user experience. diff --git a/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst b/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst deleted file mode 100644 index 528e90ed134..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-21-23-45-02.bpo-41052.3N7J2J.rst +++ /dev/null @@ -1,2 +0,0 @@ -Pickling heap types implemented in C with protocols 0 and 1 raises now an -error instead of producing incorrect data. diff --git a/Misc/NEWS.d/next/Library/2020-10-23-08-54-47.bpo-41490.-Yk6OD.rst b/Misc/NEWS.d/next/Library/2020-10-23-08-54-47.bpo-41490.-Yk6OD.rst deleted file mode 100644 index 5ad6af3d148..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-23-08-54-47.bpo-41490.-Yk6OD.rst +++ /dev/null @@ -1,3 +0,0 @@ -In ``importlib.resources``, ``.path`` method is more aggressive about -releasing handles to zipfile objects early, enabling use-cases like certifi -to leave the context open but delete the underlying zip file. diff --git a/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst b/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst deleted file mode 100644 index 4eb694c16a0..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-23-19-20-14.bpo-42103.C5obK2.rst +++ /dev/null @@ -1,3 +0,0 @@ -:exc:`~plistlib.InvalidFileException` and :exc:`RecursionError` are now -the only errors caused by loading malformed binary Plist file (previously -ValueError and TypeError could be raised in some specific cases). diff --git a/Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst b/Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst deleted file mode 100644 index 041809803db..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-25-19-25-02.bpo-42146.6A8uvS.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix memory leak in :func:`subprocess.Popen` in case an uid (gid) specified in -`user` (`group`, `extra_groups`) overflows `uid_t` (`gid_t`). diff --git a/Misc/NEWS.d/next/Library/2020-10-26-19-08-07.bpo-42157.Bdpa04.rst b/Misc/NEWS.d/next/Library/2020-10-26-19-08-07.bpo-42157.Bdpa04.rst deleted file mode 100644 index 68778906bbb..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-26-19-08-07.bpo-42157.Bdpa04.rst +++ /dev/null @@ -1,4 +0,0 @@ -Convert the :mod:`unicodedata` extension module to the multiphase -initialization API (:pep:`489`) and convert the ``unicodedata.UCD`` -static type to a heap type. -Patch by Mohamed Koubaa and Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2020-10-26-23-29-16.bpo-42157.4wuwTe.rst b/Misc/NEWS.d/next/Library/2020-10-26-23-29-16.bpo-42157.4wuwTe.rst deleted file mode 100644 index 39365677ecd..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-26-23-29-16.bpo-42157.4wuwTe.rst +++ /dev/null @@ -1,3 +0,0 @@ -Removed the ``unicodedata.ucnhash_CAPI`` attribute which was an internal -PyCapsule object. The related private ``_PyUnicode_Name_CAPI`` structure was -moved to the internal C API. Patch by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst b/Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst deleted file mode 100644 index d8832401c93..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-31-01-16-49.bpo-26789.9BdNAt.rst +++ /dev/null @@ -1,4 +0,0 @@ -The :class:`logging.FileHandler` class now keeps a reference to the builtin -:func:`open` function to be able to open or reopen the file during Python -finalization. Fix errors like: ``NameError: name 'open' is not defined``. Patch -by Victor Stinner. diff --git a/Misc/NEWS.d/next/Library/2020-10-31-13-28-36.bpo-29566.6aDbty.rst b/Misc/NEWS.d/next/Library/2020-10-31-13-28-36.bpo-29566.6aDbty.rst deleted file mode 100644 index d54c7146885..00000000000 --- a/Misc/NEWS.d/next/Library/2020-10-31-13-28-36.bpo-29566.6aDbty.rst +++ /dev/null @@ -1 +0,0 @@ -``binhex.binhex()`` consisently writes macOS 9 line endings. diff --git a/Misc/NEWS.d/next/Library/2020-11-02-01-31-15.bpo-42233.YxRj-h.rst b/Misc/NEWS.d/next/Library/2020-11-02-01-31-15.bpo-42233.YxRj-h.rst deleted file mode 100644 index aad4249fa16..00000000000 --- a/Misc/NEWS.d/next/Library/2020-11-02-01-31-15.bpo-42233.YxRj-h.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :func:`repr` of :mod:`typing` types containing -:ref:`Generic Alias Types ` previously did not show the -parameterized types in the ``GenericAlias``. They have now been changed to do so. diff --git a/Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst b/Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst deleted file mode 100644 index e72c7d277a1..00000000000 --- a/Misc/NEWS.d/next/Library/2020-11-02-14-10-48.bpo-35455.Q1xTIo.rst +++ /dev/null @@ -1,3 +0,0 @@ -On Solaris, :func:`~time.thread_time` is now implemented with -``gethrvtime()`` because ``clock_gettime(CLOCK_THREAD_CPUTIME_ID)`` is not -always available. Patch by Jakub Kulik. diff --git a/Misc/NEWS.d/next/Security/2020-10-19-10-56-27.bpo-42051.EU_B7u.rst b/Misc/NEWS.d/next/Security/2020-10-19-10-56-27.bpo-42051.EU_B7u.rst deleted file mode 100644 index e865ed12a03..00000000000 --- a/Misc/NEWS.d/next/Security/2020-10-19-10-56-27.bpo-42051.EU_B7u.rst +++ /dev/null @@ -1,3 +0,0 @@ -The :mod:`plistlib` module no longer accepts entity declarations in XML -plist files to avoid XML vulnerabilities. This should not affect users as -entity declarations are not used in regular plist files. diff --git a/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst b/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst deleted file mode 100644 index 15d7b6549ed..00000000000 --- a/Misc/NEWS.d/next/Security/2020-10-23-19-19-30.bpo-42103.cILT66.rst +++ /dev/null @@ -1,2 +0,0 @@ -Prevented potential DoS attack via CPU and RAM exhaustion when processing -malformed Apple Property List files in binary format. diff --git a/Misc/NEWS.d/next/Tests/2020-08-03-13-44-37.bpo-41306.VDoWXI.rst b/Misc/NEWS.d/next/Tests/2020-08-03-13-44-37.bpo-41306.VDoWXI.rst deleted file mode 100644 index 5e9ba2d8a27..00000000000 --- a/Misc/NEWS.d/next/Tests/2020-08-03-13-44-37.bpo-41306.VDoWXI.rst +++ /dev/null @@ -1 +0,0 @@ -Fixed a failure in ``test_tk.test_widgets.ScaleTest`` happening when executing the test with Tk 8.6.10. diff --git a/Misc/NEWS.d/next/Tests/2020-10-05-17-43-46.bpo-41944.rf1dYb.rst b/Misc/NEWS.d/next/Tests/2020-10-05-17-43-46.bpo-41944.rf1dYb.rst deleted file mode 100644 index 4f9782f1c85..00000000000 --- a/Misc/NEWS.d/next/Tests/2020-10-05-17-43-46.bpo-41944.rf1dYb.rst +++ /dev/null @@ -1 +0,0 @@ -Tests for CJK codecs no longer call ``eval()`` on content received via HTTP. diff --git a/Misc/NEWS.d/next/Tests/2020-10-08-14-00-17.bpo-41970.aZ8QFf.rst b/Misc/NEWS.d/next/Tests/2020-10-08-14-00-17.bpo-41970.aZ8QFf.rst deleted file mode 100644 index 4cdca197fbf..00000000000 --- a/Misc/NEWS.d/next/Tests/2020-10-08-14-00-17.bpo-41970.aZ8QFf.rst +++ /dev/null @@ -1,2 +0,0 @@ -Avoid a test failure in ``test_lib2to3`` if the module has already imported -at the time the test executes. Patch by Pablo Galindo. diff --git a/Misc/NEWS.d/next/Tests/2020-10-12-00-11-47.bpo-41739.wSCc4K.rst b/Misc/NEWS.d/next/Tests/2020-10-12-00-11-47.bpo-41739.wSCc4K.rst deleted file mode 100644 index 7aee2b94444..00000000000 --- a/Misc/NEWS.d/next/Tests/2020-10-12-00-11-47.bpo-41739.wSCc4K.rst +++ /dev/null @@ -1,2 +0,0 @@ -Fix test_logging.test_race_between_set_target_and_flush(): the test now -waits until all threads complete to avoid leaking running threads. diff --git a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst b/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst deleted file mode 100644 index c45aa130914..00000000000 --- a/Misc/NEWS.d/next/Windows/2020-05-30-02-46-43.bpo-38324.476M-5.rst +++ /dev/null @@ -1 +0,0 @@ -Avoid Unicode errors when accessing certain locale data on Windows. diff --git a/Misc/NEWS.d/next/Windows/2020-08-26-09-35-06.bpo-41557.vt00cQ.rst b/Misc/NEWS.d/next/Windows/2020-08-26-09-35-06.bpo-41557.vt00cQ.rst deleted file mode 100644 index 9d85461f009..00000000000 --- a/Misc/NEWS.d/next/Windows/2020-08-26-09-35-06.bpo-41557.vt00cQ.rst +++ /dev/null @@ -1 +0,0 @@ -Update Windows installer to use SQLite 3.33.0. diff --git a/Misc/NEWS.d/next/Windows/2020-09-24-23-09-40.bpo-39107.GbUZvD.rst b/Misc/NEWS.d/next/Windows/2020-09-24-23-09-40.bpo-39107.GbUZvD.rst deleted file mode 100644 index 889ad481404..00000000000 --- a/Misc/NEWS.d/next/Windows/2020-09-24-23-09-40.bpo-39107.GbUZvD.rst +++ /dev/null @@ -1 +0,0 @@ -Update Tcl and Tk to 8.6.10 in Windows installer. diff --git a/Misc/NEWS.d/next/Windows/2020-10-18-18-43-45.bpo-38252.7Nlepg.rst b/Misc/NEWS.d/next/Windows/2020-10-18-18-43-45.bpo-38252.7Nlepg.rst deleted file mode 100644 index c103e6cfcc7..00000000000 --- a/Misc/NEWS.d/next/Windows/2020-10-18-18-43-45.bpo-38252.7Nlepg.rst +++ /dev/null @@ -1 +0,0 @@ -Use 8-byte step to detect ASCII sequence in 64-bit Windows build. diff --git a/Misc/NEWS.d/next/Windows/2020-10-20-13-19-42.bpo-38439.eMLi-t.rst b/Misc/NEWS.d/next/Windows/2020-10-20-13-19-42.bpo-38439.eMLi-t.rst deleted file mode 100644 index acbc80c10f5..00000000000 --- a/Misc/NEWS.d/next/Windows/2020-10-20-13-19-42.bpo-38439.eMLi-t.rst +++ /dev/null @@ -1 +0,0 @@ -Updates the icons for IDLE in the Windows Store package. diff --git a/Misc/NEWS.d/next/macOS/2020-10-19-12-25-19.bpo-41471.gwA7un.rst b/Misc/NEWS.d/next/macOS/2020-10-19-12-25-19.bpo-41471.gwA7un.rst deleted file mode 100644 index db5dd00b19b..00000000000 --- a/Misc/NEWS.d/next/macOS/2020-10-19-12-25-19.bpo-41471.gwA7un.rst +++ /dev/null @@ -1 +0,0 @@ -Ignore invalid prefix lengths in system proxy excludes. diff --git a/README.rst b/README.rst index 33ccfc01d9c..b27854d9178 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.10.0 alpha 1 +This is Python version 3.10.0 alpha 2 ===================================== .. image:: https://travis-ci.com/python/cpython.svg?branch=master From 74b4eda98b650245216292ace8d7e30d3997baa7 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Tue, 3 Nov 2020 10:38:31 +0100 Subject: [PATCH 11/61] bpo-40077: Convert mmap.mmap static type to a heap type (GH-23108) --- .../2020-11-02-14-39-48.bpo-40077.grY9TG.rst | 1 + Modules/mmapmodule.c | 158 ++++++++++-------- 2 files changed, 91 insertions(+), 68 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst new file mode 100644 index 00000000000..48565a5a0da --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-02-14-39-48.bpo-40077.grY9TG.rst @@ -0,0 +1 @@ +Convert :mod:`mmap` to use heap types. diff --git a/Modules/mmapmodule.c b/Modules/mmapmodule.c index 463bd40e78f..1e9684970f5 100644 --- a/Modules/mmapmodule.c +++ b/Modules/mmapmodule.c @@ -20,6 +20,7 @@ #define PY_SSIZE_T_CLEAN #include +#include "structmember.h" // PyMemberDef #include // offsetof() #ifndef MS_WINDOWS @@ -113,10 +114,23 @@ typedef struct { access_mode access; } mmap_object; +typedef struct { + PyTypeObject *mmap_object_type; +} mmap_state; + +static mmap_state * +get_mmap_state(PyObject *module) +{ + mmap_state *state = PyModule_GetState(module); + assert(state); + return state; +} static void mmap_object_dealloc(mmap_object *m_obj) { + PyTypeObject *tp = Py_TYPE(m_obj); + #ifdef MS_WINDOWS Py_BEGIN_ALLOW_THREADS if (m_obj->data != NULL) @@ -142,7 +156,9 @@ mmap_object_dealloc(mmap_object *m_obj) if (m_obj->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) m_obj); - Py_TYPE(m_obj)->tp_free((PyObject*)m_obj); + + tp->tp_free(m_obj); + Py_DECREF(tp); } static PyObject * @@ -793,6 +809,11 @@ mmap_madvise_method(mmap_object *self, PyObject *args) } #endif // HAVE_MADVISE +static struct PyMemberDef mmap_object_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(mmap_object, weakreflist), READONLY}, + {NULL}, +}; + static struct PyMethodDef mmap_object_methods[] = { {"close", (PyCFunction) mmap_close_method, METH_NOARGS}, {"find", (PyCFunction) mmap_find_method, METH_VARARGS}, @@ -1035,27 +1056,6 @@ mmap_ass_subscript(mmap_object *self, PyObject *item, PyObject *value) } } -static PySequenceMethods mmap_as_sequence = { - (lenfunc)mmap_length, /*sq_length*/ - 0, /*sq_concat*/ - 0, /*sq_repeat*/ - (ssizeargfunc)mmap_item, /*sq_item*/ - 0, /*sq_slice*/ - (ssizeobjargproc)mmap_ass_item, /*sq_ass_item*/ - 0, /*sq_ass_slice*/ -}; - -static PyMappingMethods mmap_as_mapping = { - (lenfunc)mmap_length, - (binaryfunc)mmap_subscript, - (objobjargproc)mmap_ass_subscript, -}; - -static PyBufferProcs mmap_as_buffer = { - (getbufferproc)mmap_buffer_getbuf, - (releasebufferproc)mmap_buffer_releasebuf, -}; - static PyObject * new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict); @@ -1083,47 +1083,39 @@ The default value is MAP_SHARED.\n\ To map anonymous memory, pass -1 as the fileno (both versions)."); -static PyTypeObject mmap_object_type = { - PyVarObject_HEAD_INIT(NULL, 0) - "mmap.mmap", /* tp_name */ - sizeof(mmap_object), /* tp_basicsize */ - 0, /* tp_itemsize */ - /* methods */ - (destructor)mmap_object_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)mmap__repr__method, /* tp_repr */ - 0, /* tp_as_number */ - &mmap_as_sequence, /* tp_as_sequence */ - &mmap_as_mapping, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - PyObject_GenericGetAttr, /* tp_getattro */ - 0, /* tp_setattro */ - &mmap_as_buffer, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */ - mmap_doc, /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - offsetof(mmap_object, weakreflist), /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - mmap_object_methods, /* tp_methods */ - 0, /* tp_members */ - mmap_object_getset, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - PyType_GenericAlloc, /* tp_alloc */ - new_mmap_object, /* tp_new */ - PyObject_Del, /* tp_free */ +static PyType_Slot mmap_object_slots[] = { + {Py_tp_new, new_mmap_object}, + {Py_tp_alloc, PyType_GenericAlloc}, + {Py_tp_dealloc, mmap_object_dealloc}, + {Py_tp_free, PyObject_Del}, + {Py_tp_repr, mmap__repr__method}, + {Py_tp_doc, (void *)mmap_doc}, + {Py_tp_methods, mmap_object_methods}, + {Py_tp_members, mmap_object_members}, + {Py_tp_getset, mmap_object_getset}, + {Py_tp_getattro, PyObject_GenericGetAttr}, + + /* as sequence */ + {Py_sq_length, mmap_length}, + {Py_sq_item, mmap_item}, + {Py_sq_ass_item, mmap_ass_item}, + + /* as mapping */ + {Py_mp_length, mmap_length}, + {Py_mp_subscript, mmap_subscript}, + {Py_mp_ass_subscript, mmap_ass_subscript}, + + /* as buffer */ + {Py_bf_getbuffer, mmap_buffer_getbuf}, + {Py_bf_releasebuffer, mmap_buffer_releasebuf}, + {0, NULL}, +}; + +static PyType_Spec mmap_object_spec = { + .name = "mmap.mmap", + .basicsize = sizeof(mmap_object), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, + .slots = mmap_object_slots, }; @@ -1509,19 +1501,46 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict) } #endif /* MS_WINDOWS */ +static int +mmap_traverse(PyObject *module, visitproc visit, void *arg) +{ + mmap_state *state = get_mmap_state(module); + Py_VISIT(state->mmap_object_type); + return 0; +} + +static int +mmap_clear(PyObject *module) +{ + mmap_state *state = get_mmap_state(module); + Py_CLEAR(state->mmap_object_type); + return 0; +} + +static void +mmap_free(void *module) +{ + mmap_clear((PyObject *)module); +} + static int mmap_exec(PyObject *module) { - if (PyType_Ready(&mmap_object_type) < 0) { - return -1; - } + mmap_state *state = get_mmap_state(module); Py_INCREF(PyExc_OSError); if (PyModule_AddObject(module, "error", PyExc_OSError) < 0) { Py_DECREF(PyExc_OSError); return -1; } - if (PyModule_AddType(module, &mmap_object_type) < 0) { + + state->mmap_object_type = (PyTypeObject *)PyType_FromModuleAndSpec(module, + &mmap_object_spec, + NULL); + if (state->mmap_object_type == NULL) { + return -1; + } + if (PyModule_AddType(module, state->mmap_object_type) < 0) { return -1; } @@ -1660,8 +1679,11 @@ static PyModuleDef_Slot mmap_slots[] = { static struct PyModuleDef mmapmodule = { PyModuleDef_HEAD_INIT, .m_name = "mmap", - .m_size = 0, + .m_size = sizeof(mmap_state), .m_slots = mmap_slots, + .m_traverse = mmap_traverse, + .m_clear = mmap_clear, + .m_free = mmap_free, }; PyMODINIT_FUNC From 212d32f45c91849c17a82750df1ac498d63976be Mon Sep 17 00:00:00 2001 From: Serhiy Storchaka Date: Tue, 3 Nov 2020 16:15:56 +0200 Subject: [PATCH 12/61] bpo-42249: Fix writing binary Plist files larger than 4 GiB. (GH-23121) --- Lib/plistlib.py | 2 +- .../next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst diff --git a/Lib/plistlib.py b/Lib/plistlib.py index 83b214e9dc4..2eeebe4c9a4 100644 --- a/Lib/plistlib.py +++ b/Lib/plistlib.py @@ -611,7 +611,7 @@ def _count_to_size(count): elif count < 1 << 16: return 2 - elif count << 1 << 32: + elif count < 1 << 32: return 4 else: diff --git a/Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst b/Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst new file mode 100644 index 00000000000..071a0fdda1f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-03-09-22-56.bpo-42249.vfNO2u.rst @@ -0,0 +1 @@ +Fixed writing binary Plist files larger than 4 GiB. From fd957c124c44441d9c5eaf61f7af8cf266bafcb1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 3 Nov 2020 18:07:15 +0100 Subject: [PATCH 13/61] bpo-41796: Call _PyAST_Fini() earlier to fix a leak (GH-23131) Call _PyAST_Fini() on all interpreters, not only on the main interpreter. Also, call it ealier to fix a reference leak. Python types contain a reference to themselves in in their PyTypeObject.tp_mro member. _PyAST_Fini() must called before the last GC collection to destroy AST types. _PyInterpreterState_Clear() now calls _PyAST_Fini(). It now also calls _PyWarnings_Fini() on subinterpeters, not only on the main interpreter. Add an assertion in AST init_types() to ensure that the _ast module is no longer used after _PyAST_Fini() has been called. --- Include/internal/pycore_pylifecycle.h | 2 +- Parser/asdl_c.py | 67 ++++++++++++++++++--------- Python/Python-ast.c | 33 ++++++++++--- Python/pylifecycle.c | 8 ---- Python/pystate.c | 11 +++-- 5 files changed, 78 insertions(+), 43 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index 6d84e37232b..cba3bbdc2b2 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -84,7 +84,7 @@ extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); extern void _PyTraceMalloc_Fini(void); extern void _PyWarnings_Fini(PyInterpreterState *interp); -extern void _PyAST_Fini(PyThreadState *tstate); +extern void _PyAST_Fini(PyInterpreterState *interp); extern PyStatus _PyGILState_Init(PyThreadState *tstate); extern void _PyGILState_Fini(PyThreadState *tstate); diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 9a833e841de..9fec7ae017c 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1015,18 +1015,35 @@ static int add_ast_fields(struct ast_state *state) """, 0, reflow=False) - self.emit("static int init_types(struct ast_state *state)",0) - self.emit("{", 0) - self.emit("if (state->initialized) return 1;", 1) - self.emit("if (init_identifiers(state) < 0) return 0;", 1) - self.emit("state->AST_type = PyType_FromSpec(&AST_type_spec);", 1) - self.emit("if (!state->AST_type) return 0;", 1) - self.emit("if (add_ast_fields(state) < 0) return 0;", 1) + self.file.write(textwrap.dedent(''' + static int + init_types(struct ast_state *state) + { + // init_types() must not be called after _PyAST_Fini() + // has been called + assert(state->initialized >= 0); + + if (state->initialized) { + return 1; + } + if (init_identifiers(state) < 0) { + return 0; + } + state->AST_type = PyType_FromSpec(&AST_type_spec); + if (!state->AST_type) { + return 0; + } + if (add_ast_fields(state) < 0) { + return 0; + } + ''')) for dfn in mod.dfns: self.visit(dfn) - self.emit("state->initialized = 1;", 1) - self.emit("return 1;", 1); - self.emit("}", 0) + self.file.write(textwrap.dedent(''' + state->initialized = 1; + return 1; + } + ''')) def visitProduct(self, prod, name): if prod.fields: @@ -1353,23 +1370,27 @@ def generate_ast_state(module_state, f): def generate_ast_fini(module_state, f): - f.write(""" -void _PyAST_Fini(PyThreadState *tstate) -{ -#ifdef Py_BUILD_CORE - struct ast_state *state = &tstate->interp->ast; -#else - struct ast_state *state = &global_ast_state; -#endif + f.write(textwrap.dedent(""" + void _PyAST_Fini(PyInterpreterState *interp) + { + #ifdef Py_BUILD_CORE + struct ast_state *state = &interp->ast; + #else + struct ast_state *state = &global_ast_state; + #endif -""") + """)) for s in module_state: f.write(" Py_CLEAR(state->" + s + ');\n') - f.write(""" - state->initialized = 0; -} + f.write(textwrap.dedent(""" + #if defined(Py_BUILD_CORE) && !defined(NDEBUG) + state->initialized = -1; + #else + state->initialized = 0; + #endif + } -""") + """)) def generate_module_def(mod, f, internal_h): diff --git a/Python/Python-ast.c b/Python/Python-ast.c index f04addbe201..a456b519514 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -261,10 +261,10 @@ get_ast_state(void) #include "Python-ast.h" #include "structmember.h" -void _PyAST_Fini(PyThreadState *tstate) +void _PyAST_Fini(PyInterpreterState *interp) { #ifdef Py_BUILD_CORE - struct ast_state *state = &tstate->interp->ast; + struct ast_state *state = &interp->ast; #else struct ast_state *state = &global_ast_state; #endif @@ -483,7 +483,11 @@ void _PyAST_Fini(PyThreadState *tstate) Py_CLEAR(state->vararg); Py_CLEAR(state->withitem_type); +#if defined(Py_BUILD_CORE) && !defined(NDEBUG) + state->initialized = -1; +#else state->initialized = 0; +#endif } static int init_identifiers(struct ast_state *state) @@ -1227,13 +1231,27 @@ static int add_ast_fields(struct ast_state *state) } -static int init_types(struct ast_state *state) + +static int +init_types(struct ast_state *state) { - if (state->initialized) return 1; - if (init_identifiers(state) < 0) return 0; + // init_types() must not be called after _PyAST_Fini() + // has been called + assert(state->initialized >= 0); + + if (state->initialized) { + return 1; + } + if (init_identifiers(state) < 0) { + return 0; + } state->AST_type = PyType_FromSpec(&AST_type_spec); - if (!state->AST_type) return 0; - if (add_ast_fields(state) < 0) return 0; + if (!state->AST_type) { + return 0; + } + if (add_ast_fields(state) < 0) { + return 0; + } state->mod_type = make_type(state, "mod", state->AST_type, NULL, 0, "mod = Module(stmt* body, type_ignore* type_ignores)\n" " | Interactive(stmt* body)\n" @@ -1902,6 +1920,7 @@ static int init_types(struct ast_state *state) TypeIgnore_fields, 2, "TypeIgnore(int lineno, string tag)"); if (!state->TypeIgnore_type) return 0; + state->initialized = 1; return 1; } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index ff58c1b9153..cad0fa7026b 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1545,12 +1545,6 @@ flush_std_files(void) static void finalize_interp_types(PyThreadState *tstate) { - // The _ast module state is shared by all interpreters. - // The state must only be cleared by the main interpreter. - if (_Py_IsMainInterpreter(tstate)) { - _PyAST_Fini(tstate); - } - _PyExc_Fini(tstate); _PyFrame_Fini(tstate); _PyAsyncGen_Fini(tstate); @@ -1591,8 +1585,6 @@ finalize_interp_clear(PyThreadState *tstate) _Py_ClearFileSystemEncoding(); } - _PyWarnings_Fini(tstate->interp); - finalize_interp_types(tstate); } diff --git a/Python/pystate.c b/Python/pystate.c index e37cbd5a657..c9882a7f74b 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -300,13 +300,16 @@ interpreter_clear(PyInterpreterState *interp, PyThreadState *tstate) Py_CLEAR(interp->after_forkers_parent); Py_CLEAR(interp->after_forkers_child); #endif - if (_PyRuntimeState_GetFinalizing(runtime) == NULL) { - _PyWarnings_Fini(interp); - } + + _PyAST_Fini(interp); + _PyWarnings_Fini(interp); + + // All Python types must be destroyed before the last GC collection. Python + // types create a reference cycle to themselves in their in their + // PyTypeObject.tp_mro member (the tuple contains the type). /* Last garbage collection on this interpreter */ _PyGC_CollectNoFail(tstate); - _PyGC_Fini(tstate); /* We don't clear sysdict and builtins until the end of this function. From a603c3d371cda76bae75836faf068e146eb6ab76 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Tue, 3 Nov 2020 17:31:32 +0000 Subject: [PATCH 14/61] Post 3.10.0a2 --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index b9bdc598ebc..b7d2a9aa2e1 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 2 /* Version as a string */ -#define PY_VERSION "3.10.0a2" +#define PY_VERSION "3.10.0a2+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 57aaaa8d2a43bddffeca4ead5632ce6652945cc0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 3 Nov 2020 22:27:12 +0100 Subject: [PATCH 15/61] Add _PyType_GetModuleByDef (GH-22835) See https://mail.python.org/archives/list/capi-sig@python.org/thread/T3P2QNLNLBRFHWSKYSTPMVEIL2EEKFJU/ for discussion. https://bugs.python.org/issue42100 --- Include/cpython/object.h | 2 ++ Modules/_testmultiphase.c | 8 ++++++- Modules/clinic/_testmultiphase.c.h | 7 ++++-- Objects/typeobject.c | 38 ++++++++++++++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) diff --git a/Include/cpython/object.h b/Include/cpython/object.h index 875a600f795..0db53c312f0 100644 --- a/Include/cpython/object.h +++ b/Include/cpython/object.h @@ -296,6 +296,8 @@ PyAPI_FUNC(PyObject *) _PyObject_LookupSpecial(PyObject *, _Py_Identifier *); PyAPI_FUNC(PyTypeObject *) _PyType_CalculateMetaclass(PyTypeObject *, PyObject *); PyAPI_FUNC(PyObject *) _PyType_GetDocFromInternalDoc(const char *, const char *); PyAPI_FUNC(PyObject *) _PyType_GetTextSignatureFromInternalDoc(const char *, const char *); +struct PyModuleDef; +PyAPI_FUNC(PyObject *) _PyType_GetModuleByDef(PyTypeObject *, struct PyModuleDef *); struct _Py_Identifier; PyAPI_FUNC(int) PyObject_Print(PyObject *, FILE *, int); diff --git a/Modules/_testmultiphase.c b/Modules/_testmultiphase.c index 8bba077be9b..ad60f32f7e7 100644 --- a/Modules/_testmultiphase.c +++ b/Modules/_testmultiphase.c @@ -121,24 +121,30 @@ static PyType_Spec Example_Type_spec = { }; +static PyModuleDef def_meth_state_access; + /*[clinic input] _testmultiphase.StateAccessType.get_defining_module cls: defining_class Return the module of the defining class. + +Also tests that result of _PyType_GetModuleByDef matches defining_class's +module. [clinic start generated code]*/ static PyObject * _testmultiphase_StateAccessType_get_defining_module_impl(StateAccessTypeObject *self, PyTypeObject *cls) -/*[clinic end generated code: output=ba2a14284a5d0921 input=946149f91cf72c0d]*/ +/*[clinic end generated code: output=ba2a14284a5d0921 input=356f999fc16e0933]*/ { PyObject *retval; retval = PyType_GetModule(cls); if (retval == NULL) { return NULL; } + assert(_PyType_GetModuleByDef(Py_TYPE(self), &def_meth_state_access) == retval); Py_INCREF(retval); return retval; } diff --git a/Modules/clinic/_testmultiphase.c.h b/Modules/clinic/_testmultiphase.c.h index 0d38c230f71..55f934be8c6 100644 --- a/Modules/clinic/_testmultiphase.c.h +++ b/Modules/clinic/_testmultiphase.c.h @@ -6,7 +6,10 @@ PyDoc_STRVAR(_testmultiphase_StateAccessType_get_defining_module__doc__, "get_defining_module($self, /)\n" "--\n" "\n" -"Return the module of the defining class."); +"Return the module of the defining class.\n" +"\n" +"Also tests that result of _PyType_GetModuleByDef matches defining_class\'s\n" +"module."); #define _TESTMULTIPHASE_STATEACCESSTYPE_GET_DEFINING_MODULE_METHODDEF \ {"get_defining_module", (PyCFunction)(void(*)(void))_testmultiphase_StateAccessType_get_defining_module, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _testmultiphase_StateAccessType_get_defining_module__doc__}, @@ -98,4 +101,4 @@ _testmultiphase_StateAccessType_get_count(StateAccessTypeObject *self, PyTypeObj exit: return return_value; } -/*[clinic end generated code: output=39eea487e94e7f5d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f01137bb3b373e14 input=a9049054013a1b77]*/ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index bd1587ace87..2daf374f170 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3158,6 +3158,44 @@ PyType_GetModuleState(PyTypeObject *type) return PyModule_GetState(m); } + +/* Get the module of the first superclass where the module has the + * given PyModuleDef. + * Implemented by walking the MRO, is relatively slow. + * + * This is internal API for experimentation within stdlib. Discussion: + * https://mail.python.org/archives/list/capi-sig@python.org/thread/T3P2QNLNLBRFHWSKYSTPMVEIL2EEKFJU/ + */ +PyObject * +_PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) +{ + assert(PyType_Check(type)); + assert(type->tp_mro); + int i; + for (i = 0; i < PyTuple_GET_SIZE(type->tp_mro); i++) { + PyObject *super = PyTuple_GET_ITEM(type->tp_mro, i); + if (!PyType_HasFeature((PyTypeObject *)super, Py_TPFLAGS_HEAPTYPE)) { + /* Currently, there's no way for static types to inherit + * from heap types, but to allow that possibility, + * we `continue` rather than `break`. + * We'll just potentially loop a few more times before throwing + * the error. + */ + continue; + } + PyHeapTypeObject *ht = (PyHeapTypeObject*)super; + if (ht->ht_module && PyModule_GetDef(ht->ht_module) == def) { + return ht->ht_module; + } + } + PyErr_Format( + PyExc_TypeError, + "_PyType_GetModuleByDef: No superclass of '%s' has the given module", + type->tp_name); + return NULL; +} + + /* Internal API to look for a name through the MRO, bypassing the method cache. This returns a borrowed reference, and might set an exception. 'error' is set to: -1: error with exception; 1: error without exception; 0: ok */ From bffb137cb5c694e152895556ac3f327240d769f5 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Tue, 3 Nov 2020 16:28:42 -0500 Subject: [PATCH 16/61] Fix incorrect links in ast docs (GH-23017) --- Doc/library/ast.rst | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst index b8688ae61a4..8a5c6ec5f12 100644 --- a/Doc/library/ast.rst +++ b/Doc/library/ast.rst @@ -80,12 +80,13 @@ Node classes end_col_offset Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`lineno`, and :attr:`col_offset` - attributes. The :attr:`lineno` and :attr:`end_lineno` are the first and - last line numbers of source text span (1-indexed so the first line is line 1) - and the :attr:`col_offset` and :attr:`end_col_offset` are the corresponding - UTF-8 byte offsets of the first and last tokens that generated the node. - The UTF-8 offset is recorded because the parser uses UTF-8 internally. + :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and + :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` + are the first and last line numbers of source text span (1-indexed so the + first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` + are the corresponding UTF-8 byte offsets of the first and last tokens that + generated the node. The UTF-8 offset is recorded because the parser uses + UTF-8 internally. Note that the end positions are not required by the compiler and are therefore optional. The end offset is *after* the last symbol, for example From db6434c474f7389a98b8118ca87fca988416bf33 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Tue, 3 Nov 2020 22:31:49 +0000 Subject: [PATCH 17/61] Enable signing of nuget.org packages and update to supported timestamp server (GH-23132) --- .azure-pipelines/windows-release/stage-pack-msix.yml | 3 ++- .azure-pipelines/windows-release/stage-pack-nuget.yml | 10 +++++++++- .azure-pipelines/windows-release/stage-sign.yml | 2 +- PCbuild/pyproject.props | 4 ++-- Tools/msi/sdktools.psm1 | 6 +++--- 5 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml index 26a5712e845..f967cfdbe32 100644 --- a/.azure-pipelines/windows-release/stage-pack-msix.yml +++ b/.azure-pipelines/windows-release/stage-pack-msix.yml @@ -120,10 +120,11 @@ jobs: artifactName: unsigned_msix downloadPath: $(Build.BinariesDirectory) + # MSIX must be signed and timestamped simultaneously - powershell: | $failed = $true foreach ($retry in 1..3) { - signtool sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "$(SigningDescription)" (gi *.msix) + signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix) if ($?) { $failed = $false break diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml index b100364820d..8dfea382c35 100644 --- a/.azure-pipelines/windows-release/stage-pack-nuget.yml +++ b/.azure-pipelines/windows-release/stage-pack-nuget.yml @@ -4,7 +4,7 @@ jobs: condition: and(succeeded(), eq(variables['DoNuget'], 'true')) pool: - vmImage: windows-2019 + name: 'Windows Release' workspace: clean: all @@ -36,6 +36,14 @@ jobs: nuget pack "$(Build.BinariesDirectory)\layout\python.nuspec" -OutputDirectory $(Build.ArtifactStagingDirectory) -NoPackageAnalysis -NonInteractive displayName: 'Create nuget package' + - powershell: | + gci *.nupkg | %{ + nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite + } + displayName: 'Sign nuget package' + workingDirectory: $(Build.ArtifactStagingDirectory) + condition: and(succeeded(), variables['SigningCertificate']) + - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: nuget' inputs: diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml index 584772af8b4..c21e1c9f2b0 100644 --- a/.azure-pipelines/windows-release/stage-sign.yml +++ b/.azure-pipelines/windows-release/stage-sign.yml @@ -57,7 +57,7 @@ jobs: $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) $failed = $true foreach ($retry in 1..10) { - signtool timestamp /t http://timestamp.verisign.com/scripts/timestamp.dll $files + signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files if ($?) { $failed = $false break diff --git a/PCbuild/pyproject.props b/PCbuild/pyproject.props index c659d14ff8d..d7762ca1bc6 100644 --- a/PCbuild/pyproject.props +++ b/PCbuild/pyproject.props @@ -176,8 +176,8 @@ public override bool Execute() { $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot81)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" - <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "Python $(PythonVersion)" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "Python $(PythonVersion)" <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" diff --git a/Tools/msi/sdktools.psm1 b/Tools/msi/sdktools.psm1 index 8081b104d85..c5973f9abc6 100644 --- a/Tools/msi/sdktools.psm1 +++ b/Tools/msi/sdktools.psm1 @@ -37,11 +37,11 @@ function Sign-File { foreach ($a in $files) { if ($certsha1) { - SignTool sign /sha1 $certsha1 /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a + SignTool sign /sha1 $certsha1 /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d $description $a } elseif ($certname) { - SignTool sign /a /n $certname /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a + SignTool sign /a /n $certname /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d $description $a } elseif ($certfile) { - SignTool sign /f $certfile /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a + SignTool sign /f $certfile /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d $description $a } } } From 0001a1b69ecda47b0406daa88c2943877580bcae Mon Sep 17 00:00:00 2001 From: Mario Corchero Date: Wed, 4 Nov 2020 10:27:43 +0100 Subject: [PATCH 18/61] bpo-42251: Add gettrace and getprofile to threading (GH-23125) This allows to retrieve the functions that were set in these two, which might differ from sys.gettrace and sys.getprofile within a thread. --- Doc/library/threading.rst | 20 ++++++++++++++++++ Doc/whatsnew/3.10.rst | 8 +++++++ Lib/test/test_threading.py | 21 +++++++++++++++++++ Lib/threading.py | 10 ++++++++- .../2020-11-03-14-15-35.bpo-42251.6TC32V.rst | 3 +++ 5 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-03-14-15-35.bpo-42251.6TC32V.rst diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst index 7eb12fe116b..e05486f7d08 100644 --- a/Doc/library/threading.rst +++ b/Doc/library/threading.rst @@ -121,6 +121,17 @@ This module defines the following functions: :meth:`~Thread.run` method is called. +.. function:: gettrace() + + .. index:: + single: trace function + single: debugger + + Get the trace function as set by :func:`settrace`. + + .. versionadded:: 3.10 + + .. function:: setprofile(func) .. index:: single: profile function @@ -130,6 +141,15 @@ This module defines the following functions: :meth:`~Thread.run` method is called. +.. function:: getprofile() + + .. index:: single: profile function + + Get the profiler function as set by :func:`setprofile`. + + .. versionadded:: 3.10 + + .. function:: stack_size([size]) Return the thread stack size used when creating new threads. The optional diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 60dee0c6bd1..89fc3007782 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -224,6 +224,14 @@ Add :data:`sys.orig_argv` attribute: the list of the original command line arguments passed to the Python executable. (Contributed by Victor Stinner in :issue:`23427`.) +threading +--------- + +Added :func:`threading.gettrace` and :func:`threading.getprofile` to +retrieve the functions set by :func:`threading.settrace` and +:func:`threading.setprofile` respectively. +(Contributed by Mario Corchero in :issue:`42251`.) + types ----- diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 2f0f3ae0946..e0e5406ac26 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -765,6 +765,27 @@ class ThreadTests(BaseTestCase): finally: sys.settrace(old_trace) + def test_gettrace(self): + def noop_trace(frame, event, arg): + # no operation + return noop_trace + old_trace = threading.gettrace() + try: + threading.settrace(noop_trace) + trace_func = threading.gettrace() + self.assertEqual(noop_trace,trace_func) + finally: + threading.settrace(old_trace) + + def test_getprofile(self): + def fn(*args): pass + old_profile = threading.getprofile() + try: + threading.setprofile(fn) + self.assertEqual(fn, threading.getprofile()) + finally: + threading.setprofile(old_profile) + @cpython_only def test_shutdown_locks(self): for daemon in (False, True): diff --git a/Lib/threading.py b/Lib/threading.py index 06c77f70fe7..d4fe649e4f0 100644 --- a/Lib/threading.py +++ b/Lib/threading.py @@ -28,7 +28,7 @@ __all__ = ['get_ident', 'active_count', 'Condition', 'current_thread', 'Event', 'Lock', 'RLock', 'Semaphore', 'BoundedSemaphore', 'Thread', 'Barrier', 'BrokenBarrierError', 'Timer', 'ThreadError', 'setprofile', 'settrace', 'local', 'stack_size', - 'excepthook', 'ExceptHookArgs'] + 'excepthook', 'ExceptHookArgs', 'gettrace', 'getprofile'] # Rename some stuff so "from threading import *" is safe _start_new_thread = _thread.start_new_thread @@ -65,6 +65,10 @@ def setprofile(func): global _profile_hook _profile_hook = func +def getprofile(): + """Get the profiler function as set by threading.setprofile().""" + return _profile_hook + def settrace(func): """Set a trace function for all threads started from the threading module. @@ -75,6 +79,10 @@ def settrace(func): global _trace_hook _trace_hook = func +def gettrace(): + """Get the trace function as set by threading.settrace().""" + return _trace_hook + # Synchronization classes Lock = _allocate_lock diff --git a/Misc/NEWS.d/next/Library/2020-11-03-14-15-35.bpo-42251.6TC32V.rst b/Misc/NEWS.d/next/Library/2020-11-03-14-15-35.bpo-42251.6TC32V.rst new file mode 100644 index 00000000000..7435c837a2c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-03-14-15-35.bpo-42251.6TC32V.rst @@ -0,0 +1,3 @@ +Added :func:`threading.gettrace` and :func:`threading.getprofile` to +retrieve the functions set by :func:`threading.settrace` and +:func:`threading.setprofile` respectively. Patch by Mario Corchero. From 3529718925f40d14ed48d281d809187bc7314a14 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 11:20:10 +0100 Subject: [PATCH 19/61] bpo-42236: os.device_encoding() respects UTF-8 Mode (GH-23119) On Unix, the os.device_encoding() function now returns 'UTF-8' rather than the device encoding if the Python UTF-8 Mode is enabled. --- Doc/library/os.rst | 8 ++++++ Lib/test/test_utf8_mode.py | 27 +++++++++++++++++++ .../2020-11-02-23-05-17.bpo-42236.aJ6ZBR.rst | 3 +++ Python/fileutils.c | 18 +++++-------- Python/initconfig.c | 13 ++++----- 5 files changed, 52 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-02-23-05-17.bpo-42236.aJ6ZBR.rst diff --git a/Doc/library/os.rst b/Doc/library/os.rst index f9f35b31243..3ffcfa04ffa 100644 --- a/Doc/library/os.rst +++ b/Doc/library/os.rst @@ -113,6 +113,8 @@ of the UTF-8 encoding: :ref:`error handler ` being enabled for :data:`sys.stdin` and :data:`sys.stdout` (:data:`sys.stderr` continues to use ``backslashreplace`` as it does in the default locale-aware mode) +* On Unix, :func:`os.device_encoding` returns ``'UTF-8'``. rather than the + device encoding. Note that the standard stream settings in UTF-8 mode can be overridden by :envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware @@ -808,6 +810,12 @@ as internal buffering of data. Return a string describing the encoding of the device associated with *fd* if it is connected to a terminal; else return :const:`None`. + On Unix, if the :ref:`Python UTF-8 Mode ` is enabled, return + ``'UTF-8'`` rather than the device encoding. + + .. versionchanged:: 3.10 + On Unix, the function now implements the Python UTF-8 Mode. + .. function:: dup(fd) diff --git a/Lib/test/test_utf8_mode.py b/Lib/test/test_utf8_mode.py index bdb93457cfc..8b6332ee227 100644 --- a/Lib/test/test_utf8_mode.py +++ b/Lib/test/test_utf8_mode.py @@ -3,11 +3,13 @@ Test the implementation of the PEP 540: the UTF-8 Mode. """ import locale +import subprocess import sys import textwrap import unittest from test import support from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support import os_helper MS_WINDOWS = (sys.platform == 'win32') @@ -250,6 +252,31 @@ class UTF8ModeTests(unittest.TestCase): out = self.get_output('-X', 'utf8', '-E', '-c', code) self.assertEqual(out, '1') + @unittest.skipIf(MS_WINDOWS, + "os.device_encoding() doesn't implement " + "the UTF-8 Mode on Windows") + def test_device_encoding(self): + # Use stdout as TTY + if not sys.stdout.isatty(): + self.skipTest("sys.stdout is not a TTY") + + filename = 'out.txt' + self.addCleanup(os_helper.unlink, filename) + + code = (f'import os, sys; fd = sys.stdout.fileno(); ' + f'out = open({filename!r}, "w", encoding="utf-8"); ' + f'print(os.isatty(fd), os.device_encoding(fd), file=out); ' + f'out.close()') + cmd = [sys.executable, '-X', 'utf8', '-c', code] + # The stdout TTY is inherited to the child process + proc = subprocess.run(cmd, text=True) + self.assertEqual(proc.returncode, 0, proc) + + # In UTF-8 Mode, device_encoding(fd) returns "UTF-8" if fd is a TTY + with open(filename, encoding="utf8") as fp: + out = fp.read().rstrip() + self.assertEqual(out, 'True UTF-8') + if __name__ == "__main__": unittest.main() diff --git a/Misc/NEWS.d/next/Library/2020-11-02-23-05-17.bpo-42236.aJ6ZBR.rst b/Misc/NEWS.d/next/Library/2020-11-02-23-05-17.bpo-42236.aJ6ZBR.rst new file mode 100644 index 00000000000..15e26203665 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-02-23-05-17.bpo-42236.aJ6ZBR.rst @@ -0,0 +1,3 @@ +On Unix, the :func:`os.device_encoding` function now returns ``'UTF-8'`` rather +than the device encoding if the :ref:`Python UTF-8 Mode ` is +enabled. diff --git a/Python/fileutils.c b/Python/fileutils.c index 5177b372882..b589d7390d4 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -55,9 +55,6 @@ get_surrogateescape(_Py_error_handler errors, int *surrogateescape) PyObject * _Py_device_encoding(int fd) { -#if defined(MS_WINDOWS) - UINT cp; -#endif int valid; _Py_BEGIN_SUPPRESS_IPH valid = isatty(fd); @@ -66,6 +63,7 @@ _Py_device_encoding(int fd) Py_RETURN_NONE; #if defined(MS_WINDOWS) + UINT cp; if (fd == 0) cp = GetConsoleCP(); else if (fd == 1 || fd == 2) @@ -74,16 +72,14 @@ _Py_device_encoding(int fd) cp = 0; /* GetConsoleCP() and GetConsoleOutputCP() return 0 if the application has no console */ - if (cp != 0) - return PyUnicode_FromFormat("cp%u", (unsigned int)cp); -#elif defined(CODESET) - { - char *codeset = nl_langinfo(CODESET); - if (codeset != NULL && codeset[0] != 0) - return PyUnicode_FromString(codeset); + if (cp == 0) { + Py_RETURN_NONE; } + + return PyUnicode_FromFormat("cp%u", (unsigned int)cp); +#else + return _Py_GetLocaleEncodingObject(); #endif - Py_RETURN_NONE; } #if !defined(_Py_FORCE_UTF8_FS_ENCODING) && !defined(MS_WINDOWS) diff --git a/Python/initconfig.c b/Python/initconfig.c index 7bb28ed01f1..15fb3e4d287 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1515,8 +1515,8 @@ config_init_stdio_encoding(PyConfig *config, { PyStatus status; - /* If Py_SetStandardStreamEncoding() have been called, use these - parameters. */ + /* If Py_SetStandardStreamEncoding() has been called, use its + arguments if they are not NULL. */ if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { status = CONFIG_SET_BYTES_STR(config, &config->stdio_encoding, _Py_StandardStreamEncoding, @@ -1535,6 +1535,7 @@ config_init_stdio_encoding(PyConfig *config, } } + // Exit if encoding and errors are defined if (config->stdio_encoding != NULL && config->stdio_errors != NULL) { return _PyStatus_OK(); } @@ -1634,12 +1635,12 @@ config_get_fs_encoding(PyConfig *config, const PyPreConfig *preconfig, if (preconfig->utf8_mode) { return PyConfig_SetString(config, fs_encoding, L"utf-8"); } - else if (_Py_GetForceASCII()) { + + if (_Py_GetForceASCII()) { return PyConfig_SetString(config, fs_encoding, L"ascii"); } - else { - return config_get_locale_encoding(config, preconfig, fs_encoding); - } + + return config_get_locale_encoding(config, preconfig, fs_encoding); #endif // !MS_WINDOWS } From 8021875bbcf7385e651def51bc597472a569042c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 13:59:15 +0100 Subject: [PATCH 20/61] bpo-1635741: Add PyModule_AddObjectRef() function (GH-23122) Added PyModule_AddObjectRef() function: similar to PyModule_AddObjectRef() but don't steal a reference to the value on success. --- Doc/c-api/module.rst | 102 +++++++++++++++--- Doc/whatsnew/3.10.rst | 5 + Include/modsupport.h | 10 +- ...2020-11-03-11-52-27.bpo-1635741.aDYJKB.rst | 3 + Python/modsupport.c | 70 ++++++------ 5 files changed, 145 insertions(+), 45 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-03-11-52-27.bpo-1635741.aDYJKB.rst diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst index 6e9474bfa40..41a705d9e99 100644 --- a/Doc/c-api/module.rst +++ b/Doc/c-api/module.rst @@ -264,7 +264,7 @@ of the following two module creation functions: instead; only use this if you are sure you need it. Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObject`. +object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. .. _multi-phase-initialization: @@ -437,26 +437,102 @@ a function called from a module execution slot (if using multi-phase initialization), can use the following functions to help initialize the module state: +.. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) + + Add an object to *module* as *name*. This is a convenience function which + can be used from the module's initialization function. + + On success, return ``0``. On error, raise an exception and return ``-1``. + + Return ``NULL`` if *value* is ``NULL``. It must be called with an exception + raised in this case. + + Example usage:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + if (obj == NULL) { + return -1; + } + int res = PyModule_AddObjectRef(module, "spam", obj); + Py_DECREF(obj); + return res; + } + + The example can also be written without checking explicitly if *obj* is + ``NULL``:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + int res = PyModule_AddObjectRef(module, "spam", obj); + Py_XDECREF(obj); + return res; + } + + Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in + this case, since *obj* can be ``NULL``. + + .. versionadded:: 3.10 + + .. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - Add an object to *module* as *name*. This is a convenience function which can - be used from the module's initialization function. This steals a reference to - *value* on success. Return ``-1`` on error, ``0`` on success. + Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to + *value* on success (if it returns ``0``). + + The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is + easy to introduce reference leaks by misusing the + :c:func:`PyModule_AddObject` function. .. note:: - Unlike other functions that steal references, ``PyModule_AddObject()`` only - decrements the reference count of *value* **on success**. + Unlike other functions that steal references, ``PyModule_AddObject()`` + only decrements the reference count of *value* **on success**. This means that its return value must be checked, and calling code must - :c:func:`Py_DECREF` *value* manually on error. Example usage:: + :c:func:`Py_DECREF` *value* manually on error. + + Example usage:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + if (obj == NULL) { + return -1; + } + if (PyModule_AddObject(module, "spam", obj) < 0) { + Py_DECREF(obj); + return -1; + } + // PyModule_AddObject() stole a reference to obj: + // Py_DECREF(obj) is not needed here + return 0; + } + + The example can also be written without checking explicitly if *obj* is + ``NULL``:: + + static int + add_spam(PyObject *module, int value) + { + PyObject *obj = PyLong_FromLong(value); + if (PyModule_AddObject(module, "spam", obj) < 0) { + Py_XDECREF(obj); + return -1; + } + // PyModule_AddObject() stole a reference to obj: + // Py_DECREF(obj) is not needed here + return 0; + } + + Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in + this case, since *obj* can be ``NULL``. - Py_INCREF(spam); - if (PyModule_AddObject(module, "spam", spam) < 0) { - Py_DECREF(module); - Py_DECREF(spam); - return NULL; - } .. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 89fc3007782..9d9284897be 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -374,6 +374,11 @@ New Features * Added :c:func:`PyUnicode_AsUTF8AndSize` to the limited C API. (Contributed by Alex Gaynor in :issue:`41784`.) +* Added :c:func:`PyModule_AddObjectRef` function: similar to + :c:func:`PyModule_AddObjectRef` but don't steal a reference to the value on + success. + (Contributed by Victor Stinner in :issue:`1635741`.) + Porting to Python 3.10 ---------------------- diff --git a/Include/modsupport.h b/Include/modsupport.h index 4c4aab65bac..f009d586bf6 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -136,7 +136,15 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( void _PyArg_Fini(void); #endif /* Py_LIMITED_API */ -PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *); +// Add an attribute with name 'name' and value 'obj' to the module 'mod. +// On success, return 0 on success. +// On error, raise an exception and return -1. +PyAPI_FUNC(int) PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value); + +// Similar to PyModule_AddObjectRef() but steal a reference to 'obj' +// (Py_DECREF(obj)) on success (if it returns 0). +PyAPI_FUNC(int) PyModule_AddObject(PyObject *mod, const char *, PyObject *value); + PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long); PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 diff --git a/Misc/NEWS.d/next/C API/2020-11-03-11-52-27.bpo-1635741.aDYJKB.rst b/Misc/NEWS.d/next/C API/2020-11-03-11-52-27.bpo-1635741.aDYJKB.rst new file mode 100644 index 00000000000..2ab1afb922f --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-03-11-52-27.bpo-1635741.aDYJKB.rst @@ -0,0 +1,3 @@ +Added :c:func:`PyModule_AddObjectRef` function: similar to +:c:func:`PyModule_AddObjectRef` but don't steal a reference to the value on +success. Patch by Victor Stinner. diff --git a/Python/modsupport.c b/Python/modsupport.c index 2dabcf38340..8655daa1fc5 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -634,56 +634,70 @@ va_build_stack(PyObject **small_stack, Py_ssize_t small_stack_len, int -PyModule_AddObject(PyObject *m, const char *name, PyObject *o) +PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value) { - PyObject *dict; - if (!PyModule_Check(m)) { + if (!PyModule_Check(mod)) { PyErr_SetString(PyExc_TypeError, - "PyModule_AddObject() needs module as first arg"); + "PyModule_AddObjectRef() first argument " + "must be a module"); return -1; } - if (!o) { - if (!PyErr_Occurred()) - PyErr_SetString(PyExc_TypeError, - "PyModule_AddObject() needs non-NULL value"); + if (!value) { + if (!PyErr_Occurred()) { + PyErr_SetString(PyExc_SystemError, + "PyModule_AddObjectRef() must be called " + "with an exception raised if value is NULL"); + } return -1; } - dict = PyModule_GetDict(m); + PyObject *dict = PyModule_GetDict(mod); if (dict == NULL) { /* Internal error -- modules must have a dict! */ PyErr_Format(PyExc_SystemError, "module '%s' has no __dict__", - PyModule_GetName(m)); + PyModule_GetName(mod)); return -1; } - if (PyDict_SetItemString(dict, name, o)) + + if (PyDict_SetItemString(dict, name, value)) { return -1; - Py_DECREF(o); + } return 0; } + +int +PyModule_AddObject(PyObject *mod, const char *name, PyObject *value) +{ + int res = PyModule_AddObjectRef(mod, name, value); + if (res == 0) { + Py_DECREF(value); + } + return res; +} + int PyModule_AddIntConstant(PyObject *m, const char *name, long value) { - PyObject *o = PyLong_FromLong(value); - if (!o) + PyObject *obj = PyLong_FromLong(value); + if (!obj) { return -1; - if (PyModule_AddObject(m, name, o) == 0) - return 0; - Py_DECREF(o); - return -1; + } + int res = PyModule_AddObjectRef(m, name, obj); + Py_DECREF(obj); + return res; } int PyModule_AddStringConstant(PyObject *m, const char *name, const char *value) { - PyObject *o = PyUnicode_FromString(value); - if (!o) + PyObject *obj = PyUnicode_FromString(value); + if (!obj) { return -1; - if (PyModule_AddObject(m, name, o) == 0) - return 0; - Py_DECREF(o); - return -1; + } + int res = PyModule_AddObjectRef(m, name, obj); + Py_DECREF(obj); + return res; } int @@ -696,11 +710,5 @@ PyModule_AddType(PyObject *module, PyTypeObject *type) const char *name = _PyType_Name(type); assert(name != NULL); - Py_INCREF(type); - if (PyModule_AddObject(module, name, (PyObject *)type) < 0) { - Py_DECREF(type); - return -1; - } - - return 0; + return PyModule_AddObjectRef(module, name, (PyObject *)type); } From cfb41e80c1ac5940ec6f2246c9ab4a3d16ef757e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 16:15:54 +0100 Subject: [PATCH 21/61] bpo-42260: Reorganize PyConfig (GH-23149) * Move orig_argv before argv * Move program_name and platlibdir with other path configuration inputs Give a name to the PyPreConfig and PyConfig structures and separate the type definitions. --- Include/cpython/initconfig.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/cpython/initconfig.h b/Include/cpython/initconfig.h index 3a38f77073d..666c1e419ca 100644 --- a/Include/cpython/initconfig.h +++ b/Include/cpython/initconfig.h @@ -41,7 +41,7 @@ PyAPI_FUNC(PyStatus) PyWideStringList_Insert(PyWideStringList *list, /* --- PyPreConfig ----------------------------------------------- */ -typedef struct { +typedef struct PyPreConfig { int _config_init; /* _PyConfigInitEnum value */ /* Parse Py_PreInitializeFromBytesArgs() arguments? @@ -128,7 +128,7 @@ PyAPI_FUNC(void) PyPreConfig_InitIsolatedConfig(PyPreConfig *config); /* --- PyConfig ---------------------------------------------- */ /* This structure is best documented in the Doc/c-api/init_config.rst file. */ -typedef struct { +typedef struct PyConfig { int _config_init; /* _PyConfigInitEnum value */ int isolated; @@ -147,8 +147,8 @@ typedef struct { wchar_t *filesystem_errors; wchar_t *pycache_prefix; int parse_argv; + PyWideStringList orig_argv; PyWideStringList argv; - wchar_t *program_name; PyWideStringList xoptions; PyWideStringList warnoptions; int site_import; @@ -169,12 +169,13 @@ typedef struct { int legacy_windows_stdio; #endif wchar_t *check_hash_pycs_mode; - PyWideStringList orig_argv; /* --- Path configuration inputs ------------ */ int pathconfig_warnings; + wchar_t *program_name; wchar_t *pythonpath_env; wchar_t *home; + wchar_t *platlibdir; /* --- Path configuration outputs ----------- */ int module_search_paths_set; @@ -185,7 +186,6 @@ typedef struct { wchar_t *base_prefix; wchar_t *exec_prefix; wchar_t *base_exec_prefix; - wchar_t *platlibdir; /* --- Parameter only used by Py_Main() ---------- */ int skip_source_first_line; From 988f1ec8d2643a0d00851903abcdae90d57ac0e6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 16:33:55 +0100 Subject: [PATCH 22/61] bpo-1635741: _contextvars uses PyModule_AddType() (GH-23147) Replace PyModule_AddObject() with PyModule_AddType() in the _contextvars module (Python-ast.c). Add also the module name to _contextvars types name. --- Modules/_contextvarsmodule.c | 21 +++------------------ Python/context.c | 6 +++--- 2 files changed, 6 insertions(+), 21 deletions(-) diff --git a/Modules/_contextvarsmodule.c b/Modules/_contextvarsmodule.c index d6d7f375d12..d13b5962c13 100644 --- a/Modules/_contextvarsmodule.c +++ b/Modules/_contextvarsmodule.c @@ -30,30 +30,15 @@ static PyMethodDef _contextvars_methods[] = { static int _contextvars_exec(PyObject *m) { - Py_INCREF(&PyContext_Type); - if (PyModule_AddObject(m, "Context", - (PyObject *)&PyContext_Type) < 0) - { - Py_DECREF(&PyContext_Type); + if (PyModule_AddType(m, &PyContext_Type) < 0) { return -1; } - - Py_INCREF(&PyContextVar_Type); - if (PyModule_AddObject(m, "ContextVar", - (PyObject *)&PyContextVar_Type) < 0) - { - Py_DECREF(&PyContextVar_Type); + if (PyModule_AddType(m, &PyContextVar_Type) < 0) { return -1; } - - Py_INCREF(&PyContextToken_Type); - if (PyModule_AddObject(m, "Token", - (PyObject *)&PyContextToken_Type) < 0) - { - Py_DECREF(&PyContextToken_Type); + if (PyModule_AddType(m, &PyContextToken_Type) < 0) { return -1; } - return 0; } diff --git a/Python/context.c b/Python/context.c index 15d8b8ea4b9..82826bf928f 100644 --- a/Python/context.c +++ b/Python/context.c @@ -703,7 +703,7 @@ static PyMappingMethods PyContext_as_mapping = { PyTypeObject PyContext_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "Context", + "_contextvars.Context", sizeof(PyContext), .tp_methods = PyContext_methods, .tp_as_mapping = &PyContext_as_mapping, @@ -1056,7 +1056,7 @@ static PyMethodDef PyContextVar_methods[] = { PyTypeObject PyContextVar_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "ContextVar", + "_contextvars.ContextVar", sizeof(PyContextVar), .tp_methods = PyContextVar_methods, .tp_members = PyContextVar_members, @@ -1197,7 +1197,7 @@ static PyMethodDef PyContextTokenType_methods[] = { PyTypeObject PyContextToken_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) - "Token", + "_contextvars.Token", sizeof(PyContextToken), .tp_methods = PyContextTokenType_methods, .tp_getset = PyContextTokenType_getsetlist, From 18ce7f1d0a3d65f34f25c5964da588743a1bfe3c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 16:37:07 +0100 Subject: [PATCH 23/61] bpo-1635741: _ast uses PyModule_AddObjectRef() (GH-23146) Replace PyModule_AddObject() with PyModule_AddObjectRef() in the _ast module (Python-ast.c). --- Parser/asdl_c.py | 6 +- Python/Python-ast.c | 515 ++++++++++++++++++-------------------------- 2 files changed, 209 insertions(+), 312 deletions(-) diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py index 9fec7ae017c..34bd7010cfb 100755 --- a/Parser/asdl_c.py +++ b/Parser/asdl_c.py @@ -1115,10 +1115,9 @@ class ASTModuleVisitor(PickleVisitor): self.emit('if (state == NULL) {', 1) self.emit('return -1;', 2) self.emit('}', 1) - self.emit('if (PyModule_AddObject(m, "AST", state->AST_type) < 0) {', 1) + self.emit('if (PyModule_AddObjectRef(m, "AST", state->AST_type) < 0) {', 1) self.emit('return -1;', 2) self.emit('}', 1) - self.emit('Py_INCREF(state->AST_type);', 1) self.emit('if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) {', 1) self.emit("return -1;", 2) self.emit('}', 1) @@ -1166,11 +1165,10 @@ PyInit__ast(void) self.addObj(cons.name) def addObj(self, name): - self.emit("if (PyModule_AddObject(m, \"%s\", " + self.emit("if (PyModule_AddObjectRef(m, \"%s\", " "state->%s_type) < 0) {" % (name, name), 1) self.emit("return -1;", 2) self.emit('}', 1) - self.emit("Py_INCREF(state->%s_type);" % name, 1) class StaticVisitor(PickleVisitor): diff --git a/Python/Python-ast.c b/Python/Python-ast.c index a456b519514..debd3e3542f 100644 --- a/Python/Python-ast.c +++ b/Python/Python-ast.c @@ -9718,10 +9718,9 @@ astmodule_exec(PyObject *m) if (state == NULL) { return -1; } - if (PyModule_AddObject(m, "AST", state->AST_type) < 0) { + if (PyModule_AddObjectRef(m, "AST", state->AST_type) < 0) { return -1; } - Py_INCREF(state->AST_type); if (PyModule_AddIntMacro(m, PyCF_ALLOW_TOP_LEVEL_AWAIT) < 0) { return -1; } @@ -9731,432 +9730,332 @@ astmodule_exec(PyObject *m) if (PyModule_AddIntMacro(m, PyCF_TYPE_COMMENTS) < 0) { return -1; } - if (PyModule_AddObject(m, "mod", state->mod_type) < 0) { + if (PyModule_AddObjectRef(m, "mod", state->mod_type) < 0) { return -1; } - Py_INCREF(state->mod_type); - if (PyModule_AddObject(m, "Module", state->Module_type) < 0) { + if (PyModule_AddObjectRef(m, "Module", state->Module_type) < 0) { return -1; } - Py_INCREF(state->Module_type); - if (PyModule_AddObject(m, "Interactive", state->Interactive_type) < 0) { + if (PyModule_AddObjectRef(m, "Interactive", state->Interactive_type) < 0) { return -1; } - Py_INCREF(state->Interactive_type); - if (PyModule_AddObject(m, "Expression", state->Expression_type) < 0) { + if (PyModule_AddObjectRef(m, "Expression", state->Expression_type) < 0) { return -1; } - Py_INCREF(state->Expression_type); - if (PyModule_AddObject(m, "FunctionType", state->FunctionType_type) < 0) { + if (PyModule_AddObjectRef(m, "FunctionType", state->FunctionType_type) < 0) + { return -1; } - Py_INCREF(state->FunctionType_type); - if (PyModule_AddObject(m, "stmt", state->stmt_type) < 0) { + if (PyModule_AddObjectRef(m, "stmt", state->stmt_type) < 0) { return -1; } - Py_INCREF(state->stmt_type); - if (PyModule_AddObject(m, "FunctionDef", state->FunctionDef_type) < 0) { + if (PyModule_AddObjectRef(m, "FunctionDef", state->FunctionDef_type) < 0) { return -1; } - Py_INCREF(state->FunctionDef_type); - if (PyModule_AddObject(m, "AsyncFunctionDef", state->AsyncFunctionDef_type) + if (PyModule_AddObjectRef(m, "AsyncFunctionDef", + state->AsyncFunctionDef_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "ClassDef", state->ClassDef_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Return", state->Return_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Delete", state->Delete_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Assign", state->Assign_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "AugAssign", state->AugAssign_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "AnnAssign", state->AnnAssign_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "For", state->For_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "AsyncFor", state->AsyncFor_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "While", state->While_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "If", state->If_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "With", state->With_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "AsyncWith", state->AsyncWith_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Raise", state->Raise_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Try", state->Try_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Assert", state->Assert_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Import", state->Import_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "ImportFrom", state->ImportFrom_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Global", state->Global_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Nonlocal", state->Nonlocal_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Expr", state->Expr_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Pass", state->Pass_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Break", state->Break_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Continue", state->Continue_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "expr", state->expr_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "BoolOp", state->BoolOp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "NamedExpr", state->NamedExpr_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "BinOp", state->BinOp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "UnaryOp", state->UnaryOp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Lambda", state->Lambda_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "IfExp", state->IfExp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Dict", state->Dict_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Set", state->Set_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "ListComp", state->ListComp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "SetComp", state->SetComp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "DictComp", state->DictComp_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "GeneratorExp", state->GeneratorExp_type) < 0) + { + return -1; + } + if (PyModule_AddObjectRef(m, "Await", state->Await_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Yield", state->Yield_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "YieldFrom", state->YieldFrom_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Compare", state->Compare_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Call", state->Call_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "FormattedValue", state->FormattedValue_type) < 0) { return -1; } - Py_INCREF(state->AsyncFunctionDef_type); - if (PyModule_AddObject(m, "ClassDef", state->ClassDef_type) < 0) { + if (PyModule_AddObjectRef(m, "JoinedStr", state->JoinedStr_type) < 0) { return -1; } - Py_INCREF(state->ClassDef_type); - if (PyModule_AddObject(m, "Return", state->Return_type) < 0) { + if (PyModule_AddObjectRef(m, "Constant", state->Constant_type) < 0) { return -1; } - Py_INCREF(state->Return_type); - if (PyModule_AddObject(m, "Delete", state->Delete_type) < 0) { + if (PyModule_AddObjectRef(m, "Attribute", state->Attribute_type) < 0) { return -1; } - Py_INCREF(state->Delete_type); - if (PyModule_AddObject(m, "Assign", state->Assign_type) < 0) { + if (PyModule_AddObjectRef(m, "Subscript", state->Subscript_type) < 0) { return -1; } - Py_INCREF(state->Assign_type); - if (PyModule_AddObject(m, "AugAssign", state->AugAssign_type) < 0) { + if (PyModule_AddObjectRef(m, "Starred", state->Starred_type) < 0) { return -1; } - Py_INCREF(state->AugAssign_type); - if (PyModule_AddObject(m, "AnnAssign", state->AnnAssign_type) < 0) { + if (PyModule_AddObjectRef(m, "Name", state->Name_type) < 0) { return -1; } - Py_INCREF(state->AnnAssign_type); - if (PyModule_AddObject(m, "For", state->For_type) < 0) { + if (PyModule_AddObjectRef(m, "List", state->List_type) < 0) { return -1; } - Py_INCREF(state->For_type); - if (PyModule_AddObject(m, "AsyncFor", state->AsyncFor_type) < 0) { + if (PyModule_AddObjectRef(m, "Tuple", state->Tuple_type) < 0) { return -1; } - Py_INCREF(state->AsyncFor_type); - if (PyModule_AddObject(m, "While", state->While_type) < 0) { + if (PyModule_AddObjectRef(m, "Slice", state->Slice_type) < 0) { return -1; } - Py_INCREF(state->While_type); - if (PyModule_AddObject(m, "If", state->If_type) < 0) { + if (PyModule_AddObjectRef(m, "expr_context", state->expr_context_type) < 0) + { return -1; } - Py_INCREF(state->If_type); - if (PyModule_AddObject(m, "With", state->With_type) < 0) { + if (PyModule_AddObjectRef(m, "Load", state->Load_type) < 0) { return -1; } - Py_INCREF(state->With_type); - if (PyModule_AddObject(m, "AsyncWith", state->AsyncWith_type) < 0) { + if (PyModule_AddObjectRef(m, "Store", state->Store_type) < 0) { return -1; } - Py_INCREF(state->AsyncWith_type); - if (PyModule_AddObject(m, "Raise", state->Raise_type) < 0) { + if (PyModule_AddObjectRef(m, "Del", state->Del_type) < 0) { return -1; } - Py_INCREF(state->Raise_type); - if (PyModule_AddObject(m, "Try", state->Try_type) < 0) { + if (PyModule_AddObjectRef(m, "boolop", state->boolop_type) < 0) { return -1; } - Py_INCREF(state->Try_type); - if (PyModule_AddObject(m, "Assert", state->Assert_type) < 0) { + if (PyModule_AddObjectRef(m, "And", state->And_type) < 0) { return -1; } - Py_INCREF(state->Assert_type); - if (PyModule_AddObject(m, "Import", state->Import_type) < 0) { + if (PyModule_AddObjectRef(m, "Or", state->Or_type) < 0) { return -1; } - Py_INCREF(state->Import_type); - if (PyModule_AddObject(m, "ImportFrom", state->ImportFrom_type) < 0) { + if (PyModule_AddObjectRef(m, "operator", state->operator_type) < 0) { return -1; } - Py_INCREF(state->ImportFrom_type); - if (PyModule_AddObject(m, "Global", state->Global_type) < 0) { + if (PyModule_AddObjectRef(m, "Add", state->Add_type) < 0) { return -1; } - Py_INCREF(state->Global_type); - if (PyModule_AddObject(m, "Nonlocal", state->Nonlocal_type) < 0) { + if (PyModule_AddObjectRef(m, "Sub", state->Sub_type) < 0) { return -1; } - Py_INCREF(state->Nonlocal_type); - if (PyModule_AddObject(m, "Expr", state->Expr_type) < 0) { + if (PyModule_AddObjectRef(m, "Mult", state->Mult_type) < 0) { return -1; } - Py_INCREF(state->Expr_type); - if (PyModule_AddObject(m, "Pass", state->Pass_type) < 0) { + if (PyModule_AddObjectRef(m, "MatMult", state->MatMult_type) < 0) { return -1; } - Py_INCREF(state->Pass_type); - if (PyModule_AddObject(m, "Break", state->Break_type) < 0) { + if (PyModule_AddObjectRef(m, "Div", state->Div_type) < 0) { return -1; } - Py_INCREF(state->Break_type); - if (PyModule_AddObject(m, "Continue", state->Continue_type) < 0) { + if (PyModule_AddObjectRef(m, "Mod", state->Mod_type) < 0) { return -1; } - Py_INCREF(state->Continue_type); - if (PyModule_AddObject(m, "expr", state->expr_type) < 0) { + if (PyModule_AddObjectRef(m, "Pow", state->Pow_type) < 0) { return -1; } - Py_INCREF(state->expr_type); - if (PyModule_AddObject(m, "BoolOp", state->BoolOp_type) < 0) { + if (PyModule_AddObjectRef(m, "LShift", state->LShift_type) < 0) { return -1; } - Py_INCREF(state->BoolOp_type); - if (PyModule_AddObject(m, "NamedExpr", state->NamedExpr_type) < 0) { + if (PyModule_AddObjectRef(m, "RShift", state->RShift_type) < 0) { return -1; } - Py_INCREF(state->NamedExpr_type); - if (PyModule_AddObject(m, "BinOp", state->BinOp_type) < 0) { + if (PyModule_AddObjectRef(m, "BitOr", state->BitOr_type) < 0) { return -1; } - Py_INCREF(state->BinOp_type); - if (PyModule_AddObject(m, "UnaryOp", state->UnaryOp_type) < 0) { + if (PyModule_AddObjectRef(m, "BitXor", state->BitXor_type) < 0) { return -1; } - Py_INCREF(state->UnaryOp_type); - if (PyModule_AddObject(m, "Lambda", state->Lambda_type) < 0) { + if (PyModule_AddObjectRef(m, "BitAnd", state->BitAnd_type) < 0) { return -1; } - Py_INCREF(state->Lambda_type); - if (PyModule_AddObject(m, "IfExp", state->IfExp_type) < 0) { + if (PyModule_AddObjectRef(m, "FloorDiv", state->FloorDiv_type) < 0) { return -1; } - Py_INCREF(state->IfExp_type); - if (PyModule_AddObject(m, "Dict", state->Dict_type) < 0) { + if (PyModule_AddObjectRef(m, "unaryop", state->unaryop_type) < 0) { return -1; } - Py_INCREF(state->Dict_type); - if (PyModule_AddObject(m, "Set", state->Set_type) < 0) { + if (PyModule_AddObjectRef(m, "Invert", state->Invert_type) < 0) { return -1; } - Py_INCREF(state->Set_type); - if (PyModule_AddObject(m, "ListComp", state->ListComp_type) < 0) { + if (PyModule_AddObjectRef(m, "Not", state->Not_type) < 0) { return -1; } - Py_INCREF(state->ListComp_type); - if (PyModule_AddObject(m, "SetComp", state->SetComp_type) < 0) { + if (PyModule_AddObjectRef(m, "UAdd", state->UAdd_type) < 0) { return -1; } - Py_INCREF(state->SetComp_type); - if (PyModule_AddObject(m, "DictComp", state->DictComp_type) < 0) { + if (PyModule_AddObjectRef(m, "USub", state->USub_type) < 0) { return -1; } - Py_INCREF(state->DictComp_type); - if (PyModule_AddObject(m, "GeneratorExp", state->GeneratorExp_type) < 0) { + if (PyModule_AddObjectRef(m, "cmpop", state->cmpop_type) < 0) { return -1; } - Py_INCREF(state->GeneratorExp_type); - if (PyModule_AddObject(m, "Await", state->Await_type) < 0) { + if (PyModule_AddObjectRef(m, "Eq", state->Eq_type) < 0) { return -1; } - Py_INCREF(state->Await_type); - if (PyModule_AddObject(m, "Yield", state->Yield_type) < 0) { + if (PyModule_AddObjectRef(m, "NotEq", state->NotEq_type) < 0) { return -1; } - Py_INCREF(state->Yield_type); - if (PyModule_AddObject(m, "YieldFrom", state->YieldFrom_type) < 0) { + if (PyModule_AddObjectRef(m, "Lt", state->Lt_type) < 0) { return -1; } - Py_INCREF(state->YieldFrom_type); - if (PyModule_AddObject(m, "Compare", state->Compare_type) < 0) { + if (PyModule_AddObjectRef(m, "LtE", state->LtE_type) < 0) { return -1; } - Py_INCREF(state->Compare_type); - if (PyModule_AddObject(m, "Call", state->Call_type) < 0) { + if (PyModule_AddObjectRef(m, "Gt", state->Gt_type) < 0) { return -1; } - Py_INCREF(state->Call_type); - if (PyModule_AddObject(m, "FormattedValue", state->FormattedValue_type) < + if (PyModule_AddObjectRef(m, "GtE", state->GtE_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "Is", state->Is_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "IsNot", state->IsNot_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "In", state->In_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "NotIn", state->NotIn_type) < 0) { + return -1; + } + if (PyModule_AddObjectRef(m, "comprehension", state->comprehension_type) < 0) { return -1; } - Py_INCREF(state->FormattedValue_type); - if (PyModule_AddObject(m, "JoinedStr", state->JoinedStr_type) < 0) { + if (PyModule_AddObjectRef(m, "excepthandler", state->excepthandler_type) < + 0) { return -1; } - Py_INCREF(state->JoinedStr_type); - if (PyModule_AddObject(m, "Constant", state->Constant_type) < 0) { + if (PyModule_AddObjectRef(m, "ExceptHandler", state->ExceptHandler_type) < + 0) { return -1; } - Py_INCREF(state->Constant_type); - if (PyModule_AddObject(m, "Attribute", state->Attribute_type) < 0) { + if (PyModule_AddObjectRef(m, "arguments", state->arguments_type) < 0) { return -1; } - Py_INCREF(state->Attribute_type); - if (PyModule_AddObject(m, "Subscript", state->Subscript_type) < 0) { + if (PyModule_AddObjectRef(m, "arg", state->arg_type) < 0) { return -1; } - Py_INCREF(state->Subscript_type); - if (PyModule_AddObject(m, "Starred", state->Starred_type) < 0) { + if (PyModule_AddObjectRef(m, "keyword", state->keyword_type) < 0) { return -1; } - Py_INCREF(state->Starred_type); - if (PyModule_AddObject(m, "Name", state->Name_type) < 0) { + if (PyModule_AddObjectRef(m, "alias", state->alias_type) < 0) { return -1; } - Py_INCREF(state->Name_type); - if (PyModule_AddObject(m, "List", state->List_type) < 0) { + if (PyModule_AddObjectRef(m, "withitem", state->withitem_type) < 0) { return -1; } - Py_INCREF(state->List_type); - if (PyModule_AddObject(m, "Tuple", state->Tuple_type) < 0) { + if (PyModule_AddObjectRef(m, "type_ignore", state->type_ignore_type) < 0) { return -1; } - Py_INCREF(state->Tuple_type); - if (PyModule_AddObject(m, "Slice", state->Slice_type) < 0) { + if (PyModule_AddObjectRef(m, "TypeIgnore", state->TypeIgnore_type) < 0) { return -1; } - Py_INCREF(state->Slice_type); - if (PyModule_AddObject(m, "expr_context", state->expr_context_type) < 0) { - return -1; - } - Py_INCREF(state->expr_context_type); - if (PyModule_AddObject(m, "Load", state->Load_type) < 0) { - return -1; - } - Py_INCREF(state->Load_type); - if (PyModule_AddObject(m, "Store", state->Store_type) < 0) { - return -1; - } - Py_INCREF(state->Store_type); - if (PyModule_AddObject(m, "Del", state->Del_type) < 0) { - return -1; - } - Py_INCREF(state->Del_type); - if (PyModule_AddObject(m, "boolop", state->boolop_type) < 0) { - return -1; - } - Py_INCREF(state->boolop_type); - if (PyModule_AddObject(m, "And", state->And_type) < 0) { - return -1; - } - Py_INCREF(state->And_type); - if (PyModule_AddObject(m, "Or", state->Or_type) < 0) { - return -1; - } - Py_INCREF(state->Or_type); - if (PyModule_AddObject(m, "operator", state->operator_type) < 0) { - return -1; - } - Py_INCREF(state->operator_type); - if (PyModule_AddObject(m, "Add", state->Add_type) < 0) { - return -1; - } - Py_INCREF(state->Add_type); - if (PyModule_AddObject(m, "Sub", state->Sub_type) < 0) { - return -1; - } - Py_INCREF(state->Sub_type); - if (PyModule_AddObject(m, "Mult", state->Mult_type) < 0) { - return -1; - } - Py_INCREF(state->Mult_type); - if (PyModule_AddObject(m, "MatMult", state->MatMult_type) < 0) { - return -1; - } - Py_INCREF(state->MatMult_type); - if (PyModule_AddObject(m, "Div", state->Div_type) < 0) { - return -1; - } - Py_INCREF(state->Div_type); - if (PyModule_AddObject(m, "Mod", state->Mod_type) < 0) { - return -1; - } - Py_INCREF(state->Mod_type); - if (PyModule_AddObject(m, "Pow", state->Pow_type) < 0) { - return -1; - } - Py_INCREF(state->Pow_type); - if (PyModule_AddObject(m, "LShift", state->LShift_type) < 0) { - return -1; - } - Py_INCREF(state->LShift_type); - if (PyModule_AddObject(m, "RShift", state->RShift_type) < 0) { - return -1; - } - Py_INCREF(state->RShift_type); - if (PyModule_AddObject(m, "BitOr", state->BitOr_type) < 0) { - return -1; - } - Py_INCREF(state->BitOr_type); - if (PyModule_AddObject(m, "BitXor", state->BitXor_type) < 0) { - return -1; - } - Py_INCREF(state->BitXor_type); - if (PyModule_AddObject(m, "BitAnd", state->BitAnd_type) < 0) { - return -1; - } - Py_INCREF(state->BitAnd_type); - if (PyModule_AddObject(m, "FloorDiv", state->FloorDiv_type) < 0) { - return -1; - } - Py_INCREF(state->FloorDiv_type); - if (PyModule_AddObject(m, "unaryop", state->unaryop_type) < 0) { - return -1; - } - Py_INCREF(state->unaryop_type); - if (PyModule_AddObject(m, "Invert", state->Invert_type) < 0) { - return -1; - } - Py_INCREF(state->Invert_type); - if (PyModule_AddObject(m, "Not", state->Not_type) < 0) { - return -1; - } - Py_INCREF(state->Not_type); - if (PyModule_AddObject(m, "UAdd", state->UAdd_type) < 0) { - return -1; - } - Py_INCREF(state->UAdd_type); - if (PyModule_AddObject(m, "USub", state->USub_type) < 0) { - return -1; - } - Py_INCREF(state->USub_type); - if (PyModule_AddObject(m, "cmpop", state->cmpop_type) < 0) { - return -1; - } - Py_INCREF(state->cmpop_type); - if (PyModule_AddObject(m, "Eq", state->Eq_type) < 0) { - return -1; - } - Py_INCREF(state->Eq_type); - if (PyModule_AddObject(m, "NotEq", state->NotEq_type) < 0) { - return -1; - } - Py_INCREF(state->NotEq_type); - if (PyModule_AddObject(m, "Lt", state->Lt_type) < 0) { - return -1; - } - Py_INCREF(state->Lt_type); - if (PyModule_AddObject(m, "LtE", state->LtE_type) < 0) { - return -1; - } - Py_INCREF(state->LtE_type); - if (PyModule_AddObject(m, "Gt", state->Gt_type) < 0) { - return -1; - } - Py_INCREF(state->Gt_type); - if (PyModule_AddObject(m, "GtE", state->GtE_type) < 0) { - return -1; - } - Py_INCREF(state->GtE_type); - if (PyModule_AddObject(m, "Is", state->Is_type) < 0) { - return -1; - } - Py_INCREF(state->Is_type); - if (PyModule_AddObject(m, "IsNot", state->IsNot_type) < 0) { - return -1; - } - Py_INCREF(state->IsNot_type); - if (PyModule_AddObject(m, "In", state->In_type) < 0) { - return -1; - } - Py_INCREF(state->In_type); - if (PyModule_AddObject(m, "NotIn", state->NotIn_type) < 0) { - return -1; - } - Py_INCREF(state->NotIn_type); - if (PyModule_AddObject(m, "comprehension", state->comprehension_type) < 0) { - return -1; - } - Py_INCREF(state->comprehension_type); - if (PyModule_AddObject(m, "excepthandler", state->excepthandler_type) < 0) { - return -1; - } - Py_INCREF(state->excepthandler_type); - if (PyModule_AddObject(m, "ExceptHandler", state->ExceptHandler_type) < 0) { - return -1; - } - Py_INCREF(state->ExceptHandler_type); - if (PyModule_AddObject(m, "arguments", state->arguments_type) < 0) { - return -1; - } - Py_INCREF(state->arguments_type); - if (PyModule_AddObject(m, "arg", state->arg_type) < 0) { - return -1; - } - Py_INCREF(state->arg_type); - if (PyModule_AddObject(m, "keyword", state->keyword_type) < 0) { - return -1; - } - Py_INCREF(state->keyword_type); - if (PyModule_AddObject(m, "alias", state->alias_type) < 0) { - return -1; - } - Py_INCREF(state->alias_type); - if (PyModule_AddObject(m, "withitem", state->withitem_type) < 0) { - return -1; - } - Py_INCREF(state->withitem_type); - if (PyModule_AddObject(m, "type_ignore", state->type_ignore_type) < 0) { - return -1; - } - Py_INCREF(state->type_ignore_type); - if (PyModule_AddObject(m, "TypeIgnore", state->TypeIgnore_type) < 0) { - return -1; - } - Py_INCREF(state->TypeIgnore_type); return 0; } From 58ca33b4674f39189b03c9a39fa7b676b43b3d08 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 17:33:06 +0100 Subject: [PATCH 24/61] bpo-1635741: Fix ref leak in _PyWarnings_Init() error path (GH-23151) Replace PyModule_AddObject() with PyModule_AddObjectRef() in the _warnings module to fix a reference leak on error. Use also PyModule_AddObjectRef() in importdl.c. --- Python/_warnings.c | 11 +++-------- Python/importdl.c | 5 ++--- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 3c048af4193..e42b7c3be3d 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1395,18 +1395,13 @@ _PyWarnings_Init(void) goto error; } - Py_INCREF(st->filters); - if (PyModule_AddObject(m, "filters", st->filters) < 0) { + if (PyModule_AddObjectRef(m, "filters", st->filters) < 0) { goto error; } - - Py_INCREF(st->once_registry); - if (PyModule_AddObject(m, "_onceregistry", st->once_registry) < 0) { + if (PyModule_AddObjectRef(m, "_onceregistry", st->once_registry) < 0) { goto error; } - - Py_INCREF(st->default_action); - if (PyModule_AddObject(m, "_defaultaction", st->default_action) < 0) { + if (PyModule_AddObjectRef(m, "_defaultaction", st->default_action) < 0) { goto error; } diff --git a/Python/importdl.c b/Python/importdl.c index fbeb9fb7540..1847eba74ae 100644 --- a/Python/importdl.c +++ b/Python/importdl.c @@ -220,10 +220,9 @@ _PyImport_LoadDynamicModuleWithSpec(PyObject *spec, FILE *fp) def->m_base.m_init = p0; /* Remember the filename as the __file__ attribute */ - if (PyModule_AddObject(m, "__file__", path) < 0) + if (PyModule_AddObjectRef(m, "__file__", path) < 0) { PyErr_Clear(); /* Not important enough to report */ - else - Py_INCREF(path); + } PyObject *modules = PyImport_GetModuleDict(); if (_PyImport_FixupExtensionObject(m, name_unicode, path, modules) < 0) From af1d64d9f7a7cf673279725fdbaf4adcca51d41f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 4 Nov 2020 17:34:34 +0100 Subject: [PATCH 25/61] bpo-42260: Main init modify sys.flags in-place (GH-23150) When Py_Initialize() is called twice, the second call now updates more sys attributes for the configuration, rather than only sys.argv. * Rename _PySys_InitMain() to _PySys_UpdateConfig(). * _PySys_UpdateConfig() now modifies sys.flags in-place, instead of creating a new flags object. * Remove old commented sys.flags flags (unbuffered and skip_first). * Add private _PySys_GetObject() function. * When Py_Initialize(), Py_InitializeFromConfig() and --- Include/internal/pycore_pylifecycle.h | 2 +- .../2020-11-04-16-31-55.bpo-42260.CmgHtF.rst | 3 + Python/pylifecycle.c | 45 +++---- Python/sysmodule.c | 115 +++++++++++------- 4 files changed, 90 insertions(+), 75 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-04-16-31-55.bpo-42260.CmgHtF.rst diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index cba3bbdc2b2..606252b968d 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -44,7 +44,7 @@ extern PyStatus _PySys_Create( PyObject **sysmod_p); extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options); extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config); -extern int _PySys_InitMain(PyThreadState *tstate); +extern int _PySys_UpdateConfig(PyThreadState *tstate); extern PyStatus _PyExc_Init(PyThreadState *tstate); extern PyStatus _PyErr_Init(void); extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod); diff --git a/Misc/NEWS.d/next/C API/2020-11-04-16-31-55.bpo-42260.CmgHtF.rst b/Misc/NEWS.d/next/C API/2020-11-04-16-31-55.bpo-42260.CmgHtF.rst new file mode 100644 index 00000000000..694dd550a8e --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-04-16-31-55.bpo-42260.CmgHtF.rst @@ -0,0 +1,3 @@ +When :c:func:`Py_Initialize` is called twice, the second call now updates +more :mod:`sys` attributes for the configuration, rather than only +:data:`sys.argv`. Patch by Victor Stinner. diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index cad0fa7026b..1f826d7f6c4 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -949,19 +949,10 @@ done: configuration. Example of bpo-34008: Py_Main() called after Py_Initialize(). */ static PyStatus -_Py_ReconfigureMainInterpreter(PyThreadState *tstate) +pyinit_main_reconfigure(PyThreadState *tstate) { - const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); - - PyObject *argv = _PyWideStringList_AsList(&config->argv); - if (argv == NULL) { - return _PyStatus_NO_MEMORY(); \ - } - - int res = PyDict_SetItemString(tstate->interp->sysdict, "argv", argv); - Py_DECREF(argv); - if (res < 0) { - return _PyStatus_ERR("fail to set sys.argv"); + if (_PySys_UpdateConfig(tstate) < 0) { + return _PyStatus_ERR("fail to update sys for the new conf"); } return _PyStatus_OK(); } @@ -995,7 +986,7 @@ init_interp_main(PyThreadState *tstate) } } - if (_PySys_InitMain(tstate) < 0) { + if (_PySys_UpdateConfig(tstate) < 0) { return _PyStatus_ERR("can't finish initializing sys"); } @@ -1100,7 +1091,7 @@ pyinit_main(PyThreadState *tstate) } if (interp->runtime->initialized) { - return _Py_ReconfigureMainInterpreter(tstate); + return pyinit_main_reconfigure(tstate); } PyStatus status = init_interp_main(tstate); @@ -1111,19 +1102,6 @@ pyinit_main(PyThreadState *tstate) } -PyStatus -_Py_InitializeMain(void) -{ - PyStatus status = _PyRuntime_Initialize(); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - _PyRuntimeState *runtime = &_PyRuntime; - PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); - return pyinit_main(tstate); -} - - PyStatus Py_InitializeFromConfig(const PyConfig *config) { @@ -1191,6 +1169,19 @@ Py_Initialize(void) } +PyStatus +_Py_InitializeMain(void) +{ + PyStatus status = _PyRuntime_Initialize(); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + _PyRuntimeState *runtime = &_PyRuntime; + PyThreadState *tstate = _PyRuntimeState_GetThreadState(runtime); + return pyinit_main(tstate); +} + + static void finalize_modules_delete_special(PyThreadState *tstate, int verbose) { diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 945e639ca57..60b24946512 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -84,17 +84,24 @@ _PySys_GetObjectId(_Py_Identifier *key) return sys_get_object_id(tstate, key); } +static PyObject * +_PySys_GetObject(PyThreadState *tstate, const char *name) +{ + PyObject *sysdict = tstate->interp->sysdict; + if (sysdict == NULL) { + return NULL; + } + return _PyDict_GetItemStringWithError(sysdict, name); +} + PyObject * PySys_GetObject(const char *name) { PyThreadState *tstate = _PyThreadState_GET(); - PyObject *sd = tstate->interp->sysdict; - if (sd == NULL) { - return NULL; - } + PyObject *exc_type, *exc_value, *exc_tb; _PyErr_Fetch(tstate, &exc_type, &exc_value, &exc_tb); - PyObject *value = _PyDict_GetItemStringWithError(sd, name); + PyObject *value = _PySys_GetObject(tstate, name); /* XXX Suppress a new exception if it was raised and restore * the old one. */ _PyErr_Restore(tstate, exc_type, exc_value, exc_tb); @@ -2464,8 +2471,6 @@ static PyStructSequence_Field flags_fields[] = { {"no_site", "-S"}, {"ignore_environment", "-E"}, {"verbose", "-v"}, - /* {"unbuffered", "-u"}, */ - /* {"skip_first", "-x"}, */ {"bytes_warning", "-b"}, {"quiet", "-q"}, {"hash_randomization", "-R"}, @@ -2482,21 +2487,27 @@ static PyStructSequence_Desc flags_desc = { 15 }; -static PyObject* -make_flags(PyThreadState *tstate) +static int +set_flags_from_config(PyObject *flags, PyThreadState *tstate) { PyInterpreterState *interp = tstate->interp; const PyPreConfig *preconfig = &interp->runtime->preconfig; const PyConfig *config = _PyInterpreterState_GetConfig(interp); - PyObject *seq = PyStructSequence_New(&FlagsType); - if (seq == NULL) { - return NULL; - } - - int pos = 0; -#define SetFlag(flag) \ - PyStructSequence_SET_ITEM(seq, pos++, PyLong_FromLong(flag)) + // _PySys_UpdateConfig() modifies sys.flags in-place: + // Py_XDECREF() is needed in this case. + Py_ssize_t pos = 0; +#define SetFlagObj(expr) \ + do { \ + PyObject *value = (expr); \ + if (value == NULL) { \ + return -1; \ + } \ + Py_XDECREF(PyStructSequence_GET_ITEM(flags, pos)); \ + PyStructSequence_SET_ITEM(flags, pos, value); \ + pos++; \ + } while (0) +#define SetFlag(expr) SetFlagObj(PyLong_FromLong(expr)) SetFlag(config->parser_debug); SetFlag(config->inspect); @@ -2507,23 +2518,34 @@ make_flags(PyThreadState *tstate) SetFlag(!config->site_import); SetFlag(!config->use_environment); SetFlag(config->verbose); - /* SetFlag(saw_unbuffered_flag); */ - /* SetFlag(skipfirstline); */ SetFlag(config->bytes_warning); SetFlag(config->quiet); SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); SetFlag(config->isolated); - PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); + SetFlagObj(PyBool_FromLong(config->dev_mode)); SetFlag(preconfig->utf8_mode); +#undef SetFlagObj #undef SetFlag + return 0; +} - if (_PyErr_Occurred(tstate)) { - Py_DECREF(seq); + +static PyObject* +make_flags(PyThreadState *tstate) +{ + PyObject *flags = PyStructSequence_New(&FlagsType); + if (flags == NULL) { return NULL; } - return seq; + + if (set_flags_from_config(flags, tstate) < 0) { + Py_DECREF(flags); + return NULL; + } + return flags; } + PyDoc_STRVAR(version_info__doc__, "sys.version_info\n\ \n\ @@ -2767,14 +2789,23 @@ _PySys_InitCore(PyThreadState *tstate, PyObject *sysdict) /* implementation */ SET_SYS("implementation", make_impl_info(version_info)); - /* flags */ + // sys.flags: updated in-place later by _PySys_UpdateConfig() if (FlagsType.tp_name == 0) { if (PyStructSequence_InitType2(&FlagsType, &flags_desc) < 0) { goto type_init_failed; } } - /* Set flags to their default values (updated by _PySys_InitMain()) */ SET_SYS("flags", make_flags(tstate)); + /* prevent user from creating new instances */ + FlagsType.tp_init = NULL; + FlagsType.tp_new = NULL; + res = PyDict_DelItemString(FlagsType.tp_dict, "__new__"); + if (res < 0) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { + goto err_occurred; + } + _PyErr_Clear(tstate); + } #if defined(MS_WINDOWS) /* getwindowsversion */ @@ -2876,8 +2907,10 @@ sys_create_xoptions_dict(const PyConfig *config) } +// Update sys attributes for a new PyConfig configuration. +// This function also adds attributes that _PySys_InitCore() didn't add. int -_PySys_InitMain(PyThreadState *tstate) +_PySys_UpdateConfig(PyThreadState *tstate) { PyObject *sysdict = tstate->interp->sysdict; const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); @@ -2914,29 +2947,17 @@ _PySys_InitMain(PyThreadState *tstate) #undef COPY_LIST #undef SET_SYS_FROM_WSTR - - /* Set flags to their final values */ - SET_SYS("flags", make_flags(tstate)); - /* prevent user from creating new instances */ - FlagsType.tp_init = NULL; - FlagsType.tp_new = NULL; - res = PyDict_DelItemString(FlagsType.tp_dict, "__new__"); - if (res < 0) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_KeyError)) { - return res; - } - _PyErr_Clear(tstate); + // sys.flags + PyObject *flags = _PySys_GetObject(tstate, "flags"); // borrowed ref + if (flags == NULL) { + return -1; + } + if (set_flags_from_config(flags, tstate) < 0) { + return -1; } SET_SYS("dont_write_bytecode", PyBool_FromLong(!config->write_bytecode)); - if (get_warnoptions(tstate) == NULL) { - return -1; - } - - if (get_xoptions(tstate) == NULL) - return -1; - if (_PyErr_Occurred(tstate)) { goto err_occurred; } @@ -2977,8 +2998,8 @@ error: } -/* Create sys module without all attributes: _PySys_InitMain() should be called - later to add remaining attributes. */ +/* Create sys module without all attributes. + _PySys_UpdateConfig() should be called later to add remaining attributes. */ PyStatus _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p) { From 7184218e186811e75be663be78e57d5302bf8af6 Mon Sep 17 00:00:00 2001 From: Mohamed Koubaa Date: Wed, 4 Nov 2020 11:37:23 -0600 Subject: [PATCH 26/61] bpo-1635741: Fix PyInit_pyexpat() error handling (GH-22489) Split PyInit_pyexpat() into sub-functions and fix reference leaks on error paths. --- Modules/pyexpat.c | 510 ++++++++++++++++++++++++++++------------------ 1 file changed, 310 insertions(+), 200 deletions(-) diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c index 73ea51385ee..7d7da568972 100644 --- a/Modules/pyexpat.c +++ b/Modules/pyexpat.c @@ -1587,18 +1587,6 @@ PyDoc_STRVAR(pyexpat_module_documentation, #define MODULE_INITFUNC PyInit_pyexpat #endif -static struct PyModuleDef pyexpatmodule = { - PyModuleDef_HEAD_INIT, - MODULE_NAME, - pyexpat_module_documentation, - -1, - pyexpat_methods, - NULL, - NULL, - NULL, - NULL -}; - static int init_handler_descrs(void) { int i; @@ -1623,210 +1611,182 @@ static int init_handler_descrs(void) return 0; } -PyMODINIT_FUNC -MODULE_INITFUNC(void) +static PyObject * +add_submodule(PyObject *mod, const char *fullname) { - PyObject *m, *d; - PyObject *errmod_name = PyUnicode_FromString(MODULE_NAME ".errors"); - PyObject *errors_module; - PyObject *modelmod_name; - PyObject *model_module; - PyObject *tmpnum, *tmpstr; - PyObject *codes_dict; - PyObject *rev_codes_dict; - int res; - static struct PyExpat_CAPI capi; - PyObject *capi_object; + const char *name = strrchr(fullname, '.') + 1; - if (errmod_name == NULL) - return NULL; - modelmod_name = PyUnicode_FromString(MODULE_NAME ".model"); - if (modelmod_name == NULL) - return NULL; - - if (PyType_Ready(&Xmlparsetype) < 0 || init_handler_descrs() < 0) - return NULL; - - /* Create the module and add the functions */ - m = PyModule_Create(&pyexpatmodule); - if (m == NULL) - return NULL; - - /* Add some symbolic constants to the module */ - if (ErrorObject == NULL) { - ErrorObject = PyErr_NewException("xml.parsers.expat.ExpatError", - NULL, NULL); - if (ErrorObject == NULL) - return NULL; - } - Py_INCREF(ErrorObject); - PyModule_AddObject(m, "error", ErrorObject); - Py_INCREF(ErrorObject); - PyModule_AddObject(m, "ExpatError", ErrorObject); - Py_INCREF(&Xmlparsetype); - PyModule_AddObject(m, "XMLParserType", (PyObject *) &Xmlparsetype); - - PyModule_AddStringConstant(m, "EXPAT_VERSION", - XML_ExpatVersion()); - { - XML_Expat_Version info = XML_ExpatVersionInfo(); - PyModule_AddObject(m, "version_info", - Py_BuildValue("(iii)", info.major, - info.minor, info.micro)); - } - /* XXX When Expat supports some way of figuring out how it was - compiled, this should check and set native_encoding - appropriately. - */ - PyModule_AddStringConstant(m, "native_encoding", "UTF-8"); - - d = PyModule_GetDict(m); - if (d == NULL) { - Py_DECREF(m); - return NULL; - } - errors_module = PyDict_GetItemWithError(d, errmod_name); - if (errors_module == NULL && !PyErr_Occurred()) { - errors_module = PyModule_New(MODULE_NAME ".errors"); - if (errors_module != NULL) { - _PyImport_SetModule(errmod_name, errors_module); - /* gives away the reference to errors_module */ - PyModule_AddObject(m, "errors", errors_module); - } - } - Py_DECREF(errmod_name); - model_module = PyDict_GetItemWithError(d, modelmod_name); - if (model_module == NULL && !PyErr_Occurred()) { - model_module = PyModule_New(MODULE_NAME ".model"); - if (model_module != NULL) { - _PyImport_SetModule(modelmod_name, model_module); - /* gives away the reference to model_module */ - PyModule_AddObject(m, "model", model_module); - } - } - Py_DECREF(modelmod_name); - if (errors_module == NULL || model_module == NULL) { - /* Don't core dump later! */ - Py_DECREF(m); + PyObject *submodule = PyModule_New(fullname); + if (submodule == NULL) { return NULL; } -#if XML_COMBINED_VERSION > 19505 - { - const XML_Feature *features = XML_GetFeatureList(); - PyObject *list = PyList_New(0); - if (list == NULL) - /* just ignore it */ - PyErr_Clear(); - else { - int i = 0; - for (; features[i].feature != XML_FEATURE_END; ++i) { - int ok; - PyObject *item = Py_BuildValue("si", features[i].name, - features[i].value); - if (item == NULL) { - Py_DECREF(list); - list = NULL; - break; - } - ok = PyList_Append(list, item); - Py_DECREF(item); - if (ok < 0) { - PyErr_Clear(); - break; - } - } - if (list != NULL) - PyModule_AddObject(m, "features", list); - } + PyObject *mod_name = PyUnicode_FromString(fullname); + if (mod_name == NULL) { + Py_DECREF(submodule); + return NULL; } -#endif - codes_dict = PyDict_New(); - rev_codes_dict = PyDict_New(); + if (_PyImport_SetModule(mod_name, submodule) < 0) { + Py_DECREF(submodule); + Py_DECREF(mod_name); + return NULL; + } + Py_DECREF(mod_name); + + /* gives away the reference to the submodule */ + if (PyModule_AddObject(mod, name, submodule) < 0) { + Py_DECREF(submodule); + return NULL; + } + + return submodule; +} + +static int +add_error(PyObject *errors_module, PyObject *codes_dict, + PyObject *rev_codes_dict, const char *name, int value) +{ + const char *error_string = XML_ErrorString(value); + if (PyModule_AddStringConstant(errors_module, name, error_string) < 0) { + return -1; + } + + PyObject *num = PyLong_FromLong(value); + if (num == NULL) { + return -1; + } + + if (PyDict_SetItemString(codes_dict, error_string, num) < 0) { + Py_DECREF(num); + return -1; + } + + PyObject *str = PyUnicode_FromString(error_string); + if (str == NULL) { + Py_DECREF(num); + return -1; + } + + int res = PyDict_SetItem(rev_codes_dict, num, str); + Py_DECREF(str); + Py_DECREF(num); + if (res < 0) { + return -1; + } + + return 0; +} + +static int +add_errors_module(PyObject *mod) +{ + PyObject *errors_module = add_submodule(mod, MODULE_NAME ".errors"); + if (errors_module == NULL) { + return -1; + } + + PyObject *codes_dict = PyDict_New(); + PyObject *rev_codes_dict = PyDict_New(); if (codes_dict == NULL || rev_codes_dict == NULL) { - Py_XDECREF(codes_dict); - Py_XDECREF(rev_codes_dict); - return NULL; + goto error; } -#define MYCONST(name) \ - if (PyModule_AddStringConstant(errors_module, #name, \ - XML_ErrorString(name)) < 0) \ - return NULL; \ - tmpnum = PyLong_FromLong(name); \ - if (tmpnum == NULL) return NULL; \ - res = PyDict_SetItemString(codes_dict, \ - XML_ErrorString(name), tmpnum); \ - if (res < 0) return NULL; \ - tmpstr = PyUnicode_FromString(XML_ErrorString(name)); \ - if (tmpstr == NULL) return NULL; \ - res = PyDict_SetItem(rev_codes_dict, tmpnum, tmpstr); \ - Py_DECREF(tmpstr); \ - Py_DECREF(tmpnum); \ - if (res < 0) return NULL; \ +#define ADD_CONST(name) do { \ + if (add_error(errors_module, codes_dict, rev_codes_dict, \ + #name, name) < 0) { \ + goto error; \ + } \ + } while(0) - MYCONST(XML_ERROR_NO_MEMORY); - MYCONST(XML_ERROR_SYNTAX); - MYCONST(XML_ERROR_NO_ELEMENTS); - MYCONST(XML_ERROR_INVALID_TOKEN); - MYCONST(XML_ERROR_UNCLOSED_TOKEN); - MYCONST(XML_ERROR_PARTIAL_CHAR); - MYCONST(XML_ERROR_TAG_MISMATCH); - MYCONST(XML_ERROR_DUPLICATE_ATTRIBUTE); - MYCONST(XML_ERROR_JUNK_AFTER_DOC_ELEMENT); - MYCONST(XML_ERROR_PARAM_ENTITY_REF); - MYCONST(XML_ERROR_UNDEFINED_ENTITY); - MYCONST(XML_ERROR_RECURSIVE_ENTITY_REF); - MYCONST(XML_ERROR_ASYNC_ENTITY); - MYCONST(XML_ERROR_BAD_CHAR_REF); - MYCONST(XML_ERROR_BINARY_ENTITY_REF); - MYCONST(XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF); - MYCONST(XML_ERROR_MISPLACED_XML_PI); - MYCONST(XML_ERROR_UNKNOWN_ENCODING); - MYCONST(XML_ERROR_INCORRECT_ENCODING); - MYCONST(XML_ERROR_UNCLOSED_CDATA_SECTION); - MYCONST(XML_ERROR_EXTERNAL_ENTITY_HANDLING); - MYCONST(XML_ERROR_NOT_STANDALONE); - MYCONST(XML_ERROR_UNEXPECTED_STATE); - MYCONST(XML_ERROR_ENTITY_DECLARED_IN_PE); - MYCONST(XML_ERROR_FEATURE_REQUIRES_XML_DTD); - MYCONST(XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING); + ADD_CONST(XML_ERROR_NO_MEMORY); + ADD_CONST(XML_ERROR_SYNTAX); + ADD_CONST(XML_ERROR_NO_ELEMENTS); + ADD_CONST(XML_ERROR_INVALID_TOKEN); + ADD_CONST(XML_ERROR_UNCLOSED_TOKEN); + ADD_CONST(XML_ERROR_PARTIAL_CHAR); + ADD_CONST(XML_ERROR_TAG_MISMATCH); + ADD_CONST(XML_ERROR_DUPLICATE_ATTRIBUTE); + ADD_CONST(XML_ERROR_JUNK_AFTER_DOC_ELEMENT); + ADD_CONST(XML_ERROR_PARAM_ENTITY_REF); + ADD_CONST(XML_ERROR_UNDEFINED_ENTITY); + ADD_CONST(XML_ERROR_RECURSIVE_ENTITY_REF); + ADD_CONST(XML_ERROR_ASYNC_ENTITY); + ADD_CONST(XML_ERROR_BAD_CHAR_REF); + ADD_CONST(XML_ERROR_BINARY_ENTITY_REF); + ADD_CONST(XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF); + ADD_CONST(XML_ERROR_MISPLACED_XML_PI); + ADD_CONST(XML_ERROR_UNKNOWN_ENCODING); + ADD_CONST(XML_ERROR_INCORRECT_ENCODING); + ADD_CONST(XML_ERROR_UNCLOSED_CDATA_SECTION); + ADD_CONST(XML_ERROR_EXTERNAL_ENTITY_HANDLING); + ADD_CONST(XML_ERROR_NOT_STANDALONE); + ADD_CONST(XML_ERROR_UNEXPECTED_STATE); + ADD_CONST(XML_ERROR_ENTITY_DECLARED_IN_PE); + ADD_CONST(XML_ERROR_FEATURE_REQUIRES_XML_DTD); + ADD_CONST(XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING); /* Added in Expat 1.95.7. */ - MYCONST(XML_ERROR_UNBOUND_PREFIX); + ADD_CONST(XML_ERROR_UNBOUND_PREFIX); /* Added in Expat 1.95.8. */ - MYCONST(XML_ERROR_UNDECLARING_PREFIX); - MYCONST(XML_ERROR_INCOMPLETE_PE); - MYCONST(XML_ERROR_XML_DECL); - MYCONST(XML_ERROR_TEXT_DECL); - MYCONST(XML_ERROR_PUBLICID); - MYCONST(XML_ERROR_SUSPENDED); - MYCONST(XML_ERROR_NOT_SUSPENDED); - MYCONST(XML_ERROR_ABORTED); - MYCONST(XML_ERROR_FINISHED); - MYCONST(XML_ERROR_SUSPEND_PE); + ADD_CONST(XML_ERROR_UNDECLARING_PREFIX); + ADD_CONST(XML_ERROR_INCOMPLETE_PE); + ADD_CONST(XML_ERROR_XML_DECL); + ADD_CONST(XML_ERROR_TEXT_DECL); + ADD_CONST(XML_ERROR_PUBLICID); + ADD_CONST(XML_ERROR_SUSPENDED); + ADD_CONST(XML_ERROR_NOT_SUSPENDED); + ADD_CONST(XML_ERROR_ABORTED); + ADD_CONST(XML_ERROR_FINISHED); + ADD_CONST(XML_ERROR_SUSPEND_PE); +#undef ADD_CONST if (PyModule_AddStringConstant(errors_module, "__doc__", "Constants used to describe " - "error conditions.") < 0) - return NULL; + "error conditions.") < 0) { + goto error; + } - if (PyModule_AddObject(errors_module, "codes", codes_dict) < 0) - return NULL; - if (PyModule_AddObject(errors_module, "messages", rev_codes_dict) < 0) - return NULL; + Py_INCREF(codes_dict); + if (PyModule_AddObject(errors_module, "codes", codes_dict) < 0) { + Py_DECREF(codes_dict); + goto error; + } + Py_CLEAR(codes_dict); -#undef MYCONST + Py_INCREF(rev_codes_dict); + if (PyModule_AddObject(errors_module, "messages", rev_codes_dict) < 0) { + Py_DECREF(rev_codes_dict); + goto error; + } + Py_CLEAR(rev_codes_dict); -#define MYCONST(c) PyModule_AddIntConstant(m, #c, c) - MYCONST(XML_PARAM_ENTITY_PARSING_NEVER); - MYCONST(XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE); - MYCONST(XML_PARAM_ENTITY_PARSING_ALWAYS); -#undef MYCONST + return 0; -#define MYCONST(c) PyModule_AddIntConstant(model_module, #c, c) - PyModule_AddStringConstant(model_module, "__doc__", - "Constants used to interpret content model information."); +error: + Py_XDECREF(codes_dict); + Py_XDECREF(rev_codes_dict); + return -1; +} + +static int +add_model_module(PyObject *mod) +{ + PyObject *model_module = add_submodule(mod, MODULE_NAME ".model"); + if (model_module == NULL) { + return -1; + } + +#define MYCONST(c) do { \ + if (PyModule_AddIntConstant(model_module, #c, c) < 0) { \ + return -1; \ + } \ + } while(0) + + if (PyModule_AddStringConstant( + model_module, "__doc__", + "Constants used to interpret content model information.") < 0) { + return -1; + } MYCONST(XML_CTYPE_EMPTY); MYCONST(XML_CTYPE_ANY); @@ -1840,7 +1800,128 @@ MODULE_INITFUNC(void) MYCONST(XML_CQUANT_REP); MYCONST(XML_CQUANT_PLUS); #undef MYCONST + return 0; +} +#if XML_COMBINED_VERSION > 19505 +static int +add_features(PyObject *mod) +{ + PyObject *list = PyList_New(0); + if (list == NULL) { + return -1; + } + + const XML_Feature *features = XML_GetFeatureList(); + for (size_t i = 0; features[i].feature != XML_FEATURE_END; ++i) { + PyObject *item = Py_BuildValue("si", features[i].name, + features[i].value); + if (item == NULL) { + goto error; + } + int ok = PyList_Append(list, item); + Py_DECREF(item); + if (ok < 0) { + goto error; + } + } + if (PyModule_AddObject(mod, "features", list) < 0) { + goto error; + } + return 0; + +error: + Py_DECREF(list); + return -1; +} +#endif + +static int +pyexpat_exec(PyObject *mod) +{ + if (PyType_Ready(&Xmlparsetype) < 0) { + return -1; + } + + if (init_handler_descrs() < 0) { + return -1; + } + + /* Add some symbolic constants to the module */ + if (ErrorObject == NULL) { + ErrorObject = PyErr_NewException("xml.parsers.expat.ExpatError", + NULL, NULL); + } + if (ErrorObject == NULL) { + return -1; + } + + Py_INCREF(ErrorObject); + if (PyModule_AddObject(mod, "error", ErrorObject) < 0) { + Py_DECREF(ErrorObject); + return -1; + } + Py_INCREF(ErrorObject); + if (PyModule_AddObject(mod, "ExpatError", ErrorObject) < 0) { + Py_DECREF(ErrorObject); + return -1; + } + Py_INCREF(&Xmlparsetype); + if (PyModule_AddObject(mod, "XMLParserType", + (PyObject *) &Xmlparsetype) < 0) { + Py_DECREF(&Xmlparsetype); + return -1; + } + + if (PyModule_AddStringConstant(mod, "EXPAT_VERSION", + XML_ExpatVersion()) < 0) { + return -1; + } + { + XML_Expat_Version info = XML_ExpatVersionInfo(); + PyObject *versionInfo = Py_BuildValue("(iii)", + info.major, + info.minor, + info.micro); + if (PyModule_AddObject(mod, "version_info", versionInfo) < 0) { + Py_DECREF(versionInfo); + return -1; + } + } + /* XXX When Expat supports some way of figuring out how it was + compiled, this should check and set native_encoding + appropriately. + */ + if (PyModule_AddStringConstant(mod, "native_encoding", "UTF-8") < 0) { + return -1; + } + + if (add_errors_module(mod) < 0) { + return -1; + } + + if (add_model_module(mod) < 0) { + return -1; + } + +#if XML_COMBINED_VERSION > 19505 + if (add_features(mod) < 0) { + return -1; + } +#endif + +#define MYCONST(c) do { \ + if (PyModule_AddIntConstant(mod, #c, c) < 0) { \ + return -1; \ + } \ + } while(0) + + MYCONST(XML_PARAM_ENTITY_PARSING_NEVER); + MYCONST(XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE); + MYCONST(XML_PARAM_ENTITY_PARSING_ALWAYS); +#undef MYCONST + + static struct PyExpat_CAPI capi; /* initialize pyexpat dispatch table */ capi.size = sizeof(capi); capi.magic = PyExpat_CAPI_MAGIC; @@ -1872,10 +1953,39 @@ MODULE_INITFUNC(void) #endif /* export using capsule */ - capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); - if (capi_object) - PyModule_AddObject(m, "expat_CAPI", capi_object); - return m; + PyObject *capi_object = PyCapsule_New(&capi, PyExpat_CAPSULE_NAME, NULL); + if (capi_object == NULL) { + return -1; + } + + if (PyModule_AddObject(mod, "expat_CAPI", capi_object) < 0) { + Py_DECREF(capi_object); + return -1; + } + + return 0; +} + +static struct PyModuleDef pyexpatmodule = { + PyModuleDef_HEAD_INIT, + .m_name = MODULE_NAME, + .m_doc = pyexpat_module_documentation, + .m_size = -1, + .m_methods = pyexpat_methods, +}; + +PyMODINIT_FUNC +PyInit_pyexpat(void) +{ + PyObject *mod = PyModule_Create(&pyexpatmodule); + if (mod == NULL) + return NULL; + + if (pyexpat_exec(mod) < 0) { + Py_DECREF(mod); + return NULL; + } + return mod; } static void From 789359f47c2a744caa9a405131706099fd7ad6bd Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Wed, 4 Nov 2020 20:31:51 +0100 Subject: [PATCH 27/61] bpo-1635741: _sqlite3 uses PyModule_AddObjectRef() (GH-23148) --- Modules/_sqlite/microprotocols.c | 8 +++----- Modules/_sqlite/module.c | 25 +++++++++++++------------ 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/Modules/_sqlite/microprotocols.c b/Modules/_sqlite/microprotocols.c index cf1fefd6718..41f086791ea 100644 --- a/Modules/_sqlite/microprotocols.c +++ b/Modules/_sqlite/microprotocols.c @@ -43,12 +43,10 @@ pysqlite_microprotocols_init(PyObject *module) return -1; } - if (PyModule_AddObject(module, "adapters", psyco_adapters) < 0) { - Py_DECREF(psyco_adapters); - return -1; - } + int res = PyModule_AddObjectRef(module, "adapters", psyco_adapters); + Py_DECREF(psyco_adapters); - return 0; + return res; } diff --git a/Modules/_sqlite/module.c b/Modules/_sqlite/module.c index 33324402385..9fdf51417ed 100644 --- a/Modules/_sqlite/module.c +++ b/Modules/_sqlite/module.c @@ -263,17 +263,17 @@ pysqlite_adapt_impl(PyObject *module, PyObject *obj, PyObject *proto, return pysqlite_microprotocols_adapt(obj, proto, alt); } -static void converters_init(PyObject* module) +static int converters_init(PyObject* module) { _pysqlite_converters = PyDict_New(); if (!_pysqlite_converters) { - return; + return -1; } - if (PyModule_AddObject(module, "converters", _pysqlite_converters) < 0) { - Py_DECREF(_pysqlite_converters); - } - return; + int res = PyModule_AddObjectRef(module, "converters", _pysqlite_converters); + Py_DECREF(_pysqlite_converters); + + return res; } static PyMethodDef module_methods[] = { @@ -361,8 +361,9 @@ do { \ if (!exc) { \ goto error; \ } \ - if (PyModule_AddObject(module, name, exc) < 0) { \ - Py_DECREF(exc); \ + int res = PyModule_AddObjectRef(module, name, exc); \ + Py_DECREF(exc); \ + if (res < 0) { \ goto error; \ } \ } while (0) @@ -416,9 +417,7 @@ PyMODINIT_FUNC PyInit__sqlite3(void) non-ASCII data and bytestrings to be returned for ASCII data. Now OptimizedUnicode is an alias for str, so it has no effect. */ - Py_INCREF((PyObject*)&PyUnicode_Type); - if (PyModule_AddObject(module, "OptimizedUnicode", (PyObject*)&PyUnicode_Type) < 0) { - Py_DECREF((PyObject*)&PyUnicode_Type); + if (PyModule_AddObjectRef(module, "OptimizedUnicode", (PyObject*)&PyUnicode_Type) < 0) { goto error; } @@ -441,7 +440,9 @@ PyMODINIT_FUNC PyInit__sqlite3(void) } /* initialize the default converters */ - converters_init(module); + if (converters_init(module) < 0) { + goto error; + } error: if (PyErr_Occurred()) From 100964e0310d3a2040d0db976f7984d0507b2dbd Mon Sep 17 00:00:00 2001 From: serge-sans-paille Date: Wed, 4 Nov 2020 23:01:08 +0000 Subject: [PATCH 28/61] Disable peg generator tests when building with PGO (GH-23141) Otherwise, when running the testsuite, test_peg_generator tries to compile C code using the optimized flags and fails because it cannot find the profile data. --- Lib/test/test_peg_generator/test_c_parser.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_peg_generator/test_c_parser.py b/Lib/test/test_peg_generator/test_c_parser.py index 0dffedca789..67bb8512118 100644 --- a/Lib/test/test_peg_generator/test_c_parser.py +++ b/Lib/test/test_peg_generator/test_c_parser.py @@ -1,3 +1,4 @@ +import sysconfig import textwrap import unittest from distutils.tests.support import TempdirManager @@ -8,6 +9,11 @@ from test import support from test.support import os_helper from test.support.script_helper import assert_python_ok +_py_cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') +_pgo_flag = sysconfig.get_config_var('PGO_PROF_USE_FLAG') +if _pgo_flag and _py_cflags_nodist and _pgo_flag in _py_cflags_nodist: + raise unittest.SkipTest("peg_generator test disabled under PGO build") + test_tools.skip_if_missing("peg_generator") with test_tools.imports_under_tool("peg_generator"): from pegen.grammar_parser import GeneratedParser as GrammarParser From 048a35659aa8074afe7d7d054e7cea1f8ee6d366 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2020 00:45:56 +0100 Subject: [PATCH 29/61] bpo-42260: Add _PyInterpreterState_SetConfig() (GH-23158) * Inline _PyInterpreterState_SetConfig(): replace it with _PyConfig_Copy(). * Add _PyErr_SetFromPyStatus() * Add _PyInterpreterState_GetConfigCopy() * Add a new _PyInterpreterState_SetConfig() function. * Add an unit which gets, modifies, and sets the config. --- Doc/c-api/init_config.rst | 2 + Include/cpython/pystate.h | 30 ++++++++++++ Include/internal/pycore_initconfig.h | 2 + Include/internal/pycore_interp.h | 6 --- Lib/test/test_embed.py | 9 ++++ Programs/_testembed.c | 50 ++++++++++++++++++++ Python/initconfig.c | 20 +++++++- Python/pylifecycle.c | 70 ++++++++++++++++++++++++++-- Python/pystate.c | 16 +++++-- 9 files changed, 189 insertions(+), 16 deletions(-) diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index dad1f90bea5..c957d6c0f72 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -128,6 +128,8 @@ PyStatus Initialization error with a message. + *err_msg* must not be ``NULL``. + .. c:function:: PyStatus PyStatus_NoMemory(void) Memory allocation failure (out of memory). diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 25522b4dbec..0e6cc290912 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -193,6 +193,36 @@ PyAPI_FUNC(void) _PyInterpreterState_SetEvalFrameFunc( PyAPI_FUNC(const PyConfig*) _PyInterpreterState_GetConfig(PyInterpreterState *interp); +/* Get a copy of the current interpreter configuration. + + Return 0 on success. Raise an exception and return -1 on error. + + The caller must initialize 'config', using PyConfig_InitPythonConfig() + for example. + + Python must be preinitialized to call this method. + The caller must hold the GIL. */ +PyAPI_FUNC(int) _PyInterpreterState_GetConfigCopy( + struct PyConfig *config); + +/* Set the configuration of the current interpreter. + + This function should be called during or just after the Python + initialization. + + Update the sys module with the new configuration. If the sys module was + modified directly after the Python initialization, these changes are lost. + + Some configuration like faulthandler or warnoptions can be updated in the + configuration, but don't reconfigure Python (don't enable/disable + faulthandler and don't reconfigure warnings filters). + + Return 0 on success. Raise an exception and return -1 on error. + + The configuration should come from _PyInterpreterState_GetConfigCopy(). */ +PyAPI_FUNC(int) _PyInterpreterState_SetConfig( + const struct PyConfig *config); + // Get the configuration of the currrent interpreter. // The caller must hold the GIL. PyAPI_FUNC(const PyConfig*) _Py_GetConfig(void); diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 457a005860b..df7ad779f47 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -44,6 +44,8 @@ struct pyruntimestate; #define _PyStatus_UPDATE_FUNC(err) \ do { err.func = _PyStatus_GET_FUNC(); } while (0) +PyObject* _PyErr_SetFromPyStatus(PyStatus status); + /* --- PyWideStringList ------------------------------------------------ */ #define _PyWideStringList_INIT (PyWideStringList){.length = 0, .items = NULL} diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index 9923b6b03da..4b67a86a25a 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -263,13 +263,7 @@ struct _is { struct ast_state ast; }; -/* Used by _PyImport_Cleanup() */ extern void _PyInterpreterState_ClearModules(PyInterpreterState *interp); - -extern PyStatus _PyInterpreterState_SetConfig( - PyInterpreterState *interp, - const PyConfig *config); - extern void _PyInterpreterState_Clear(PyThreadState *tstate); diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 31dc39fd9e8..36a0e77e14c 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -1394,6 +1394,15 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): self.check_all_configs("test_init_warnoptions", config, preconfig, api=API_PYTHON) + def test_init_set_config(self): + config = { + '_init_main': 0, + 'bytes_warning': 2, + 'warnoptions': ['error::BytesWarning'], + } + self.check_all_configs("test_init_set_config", config, + api=API_ISOLATED) + def test_get_argc_argv(self): self.run_embedded_interpreter("test_get_argc_argv") # ignore output diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 5aad16a6f7c..cb3a23a101e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1526,6 +1526,55 @@ static int test_init_warnoptions(void) } +static int tune_config(void) +{ + PyConfig config; + PyConfig_InitPythonConfig(&config); + if (_PyInterpreterState_GetConfigCopy(&config) < 0) { + PyConfig_Clear(&config); + PyErr_Print(); + return -1; + } + + config.bytes_warning = 2; + + if (_PyInterpreterState_SetConfig(&config) < 0) { + PyConfig_Clear(&config); + return -1; + } + PyConfig_Clear(&config); + return 0; +} + + +static int test_set_config(void) +{ + // Initialize core + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + config_set_string(&config, &config.program_name, PROGRAM_NAME); + config._init_main = 0; + config.bytes_warning = 0; + init_from_config_clear(&config); + + // Tune the configuration using _PyInterpreterState_SetConfig() + if (tune_config() < 0) { + PyErr_Print(); + return 1; + } + + // Finish initialization: main part + PyStatus status = _Py_InitializeMain(); + if (PyStatus_Exception(status)) { + Py_ExitStatusException(status); + } + + dump_config(); + Py_Finalize(); + return 0; +} + + static void configure_init_main(PyConfig *config) { wchar_t* argv[] = { @@ -1693,6 +1742,7 @@ static struct TestCase TestCases[] = { {"test_init_setpath_config", test_init_setpath_config}, {"test_init_setpythonhome", test_init_setpythonhome}, {"test_init_warnoptions", test_init_warnoptions}, + {"test_init_set_config", test_set_config}, {"test_run_main", test_run_main}, {"test_get_argc_argv", test_get_argc_argv}, diff --git a/Python/initconfig.c b/Python/initconfig.c index 15fb3e4d287..de496ac7b52 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -242,8 +242,9 @@ PyStatus PyStatus_Ok(void) PyStatus PyStatus_Error(const char *err_msg) { + assert(err_msg != NULL); return (PyStatus){._type = _PyStatus_TYPE_ERROR, - .err_msg = err_msg}; + .err_msg = err_msg}; } PyStatus PyStatus_NoMemory(void) @@ -262,6 +263,23 @@ int PyStatus_IsExit(PyStatus status) int PyStatus_Exception(PyStatus status) { return _PyStatus_EXCEPTION(status); } +PyObject* +_PyErr_SetFromPyStatus(PyStatus status) +{ + if (!_PyStatus_IS_ERROR(status)) { + PyErr_Format(PyExc_SystemError, + "%s() expects an error PyStatus", + _PyStatus_GET_FUNC()); + } + else if (status.func) { + PyErr_Format(PyExc_ValueError, "%s: %s", status.func, status.err_msg); + } + else { + PyErr_Format(PyExc_ValueError, "%s", status.err_msg); + } + return NULL; +} + /* --- PyWideStringList ------------------------------------------------ */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 1f826d7f6c4..e34d6471e17 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -428,6 +428,67 @@ _Py_SetLocaleFromEnv(int category) } +static int +interpreter_set_config(const PyConfig *config) +{ + PyThreadState *tstate = PyThreadState_Get(); + + PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; + } + + status = _PyConfig_Copy(&tstate->interp->config, config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; + } + config = &tstate->interp->config; + + if (config->_install_importlib && _Py_IsMainInterpreter(tstate)) { + status = _PyConfig_WritePathConfig(config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; + } + } + + // Update the sys module for the new configuration + if (_PySys_UpdateConfig(tstate) < 0) { + return -1; + } + return 0; +} + + +int +_PyInterpreterState_SetConfig(const PyConfig *src_config) +{ + int res = -1; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + PyStatus status = _PyConfig_Copy(&config, src_config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + goto done; + } + + status = PyConfig_Read(&config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + goto done; + } + + res = interpreter_set_config(&config); + +done: + PyConfig_Clear(&config); + return res; +} + + /* Global initializations. Can be undone by Py_Finalize(). Don't call this twice without an intervening Py_Finalize() call. @@ -462,7 +523,7 @@ pyinit_core_reconfigure(_PyRuntimeState *runtime, return status; } - status = _PyInterpreterState_SetConfig(interp, config); + status = _PyConfig_Copy(&interp->config, config); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -550,7 +611,7 @@ pycore_create_interpreter(_PyRuntimeState *runtime, return _PyStatus_ERR("can't make main interpreter"); } - PyStatus status = _PyInterpreterState_SetConfig(interp, config); + PyStatus status = _PyConfig_Copy(&interp->config, config); if (_PyStatus_EXCEPTION(status)) { return status; } @@ -917,7 +978,7 @@ pyinit_core(_PyRuntimeState *runtime, } PyConfig config; - _PyConfig_InitCompatConfig(&config); + PyConfig_InitPythonConfig(&config); status = _PyConfig_Copy(&config, src_config); if (_PyStatus_EXCEPTION(status)) { @@ -1835,7 +1896,8 @@ new_interpreter(PyThreadState **tstate_p, int isolated_subinterpreter) config = _PyInterpreterState_GetConfig(main_interp); } - status = _PyInterpreterState_SetConfig(interp, config); + + status = _PyConfig_Copy(&interp->config, config); if (_PyStatus_EXCEPTION(status)) { goto error; } diff --git a/Python/pystate.c b/Python/pystate.c index c9882a7f74b..600cc5e03a1 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -778,7 +778,7 @@ PyState_RemoveModule(struct PyModuleDef* def) return PyList_SetItem(interp->modules_by_index, index, Py_None); } -/* Used by PyImport_Cleanup() */ +// Used by finalize_modules() void _PyInterpreterState_ClearModules(PyInterpreterState *interp) { @@ -1920,11 +1920,17 @@ _PyInterpreterState_GetConfig(PyInterpreterState *interp) } -PyStatus -_PyInterpreterState_SetConfig(PyInterpreterState *interp, - const PyConfig *config) +int +_PyInterpreterState_GetConfigCopy(PyConfig *config) { - return _PyConfig_Copy(&interp->config, config); + PyInterpreterState *interp = PyInterpreterState_Get(); + + PyStatus status = _PyConfig_Copy(config, &interp->config); + if (PyStatus_Exception(status)) { + _PyErr_SetFromPyStatus(status); + return -1; + } + return 0; } From 178695b7aee7a7aacd49a3086060e06347d1e556 Mon Sep 17 00:00:00 2001 From: Kazantcev Andrey <45011689+heckad@users.noreply.github.com> Date: Thu, 5 Nov 2020 11:52:24 +0300 Subject: [PATCH 30/61] bpo-40816 Add AsyncContextDecorator class (GH-20516) Co-authored-by: Yury Selivanov --- Doc/library/contextlib.rst | 62 +++++++++++++++++++ Lib/contextlib.py | 25 +++++++- Lib/test/test_contextlib_async.py | 27 ++++++++ .../2020-05-29-15-25-41.bpo-40816.w61Pob.rst | 1 + 4 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-05-29-15-25-41.bpo-40816.w61Pob.rst diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index e42f5a93281..ee2becb8dff 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -126,6 +126,31 @@ Functions and classes provided: .. versionadded:: 3.7 + Context managers defined with :func:`asynccontextmanager` can be used + either as decorators or with :keyword:`async with` statements:: + + import time + + async def timeit(): + now = time.monotonic() + try: + yield + finally: + print(f'it took {time.monotonic() - now}s to run') + + @timeit() + async def main(): + # ... async code ... + + When used as a decorator, a new generator instance is implicitly created on + each function call. This allows the otherwise "one-shot" context managers + created by :func:`asynccontextmanager` to meet the requirement that context + managers support multiple invocations in order to be used as decorators. + + .. versionchanged:: 3.10 + Async context managers created with :func:`asynccontextmanager` can + be used as decorators. + .. function:: closing(thing) @@ -384,6 +409,43 @@ Functions and classes provided: .. versionadded:: 3.2 +.. class:: AsyncContextManager + + Similar as ContextManger only for async + + Example of ``ContextDecorator``:: + + from asyncio import run + from contextlib import AsyncContextDecorator + + class mycontext(AsyncContextDecorator): + async def __aenter__(self): + print('Starting') + return self + + async def __aexit__(self, *exc): + print('Finishing') + return False + + >>> @mycontext() + ... async def function(): + ... print('The bit in the middle') + ... + >>> run(function()) + Starting + The bit in the middle + Finishing + + >>> async def function(): + ... async with mycontext(): + ... print('The bit in the middle') + ... + >>> run(function()) + Starting + The bit in the middle + Finishing + + .. class:: ExitStack() A context manager that is designed to make it easy to programmatically diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 82ddc1497d8..56b4968118b 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -80,6 +80,22 @@ class ContextDecorator(object): return inner +class AsyncContextDecorator(object): + "A base class or mixin that enables async context managers to work as decorators." + + def _recreate_cm(self): + """Return a recreated instance of self. + """ + return self + + def __call__(self, func): + @wraps(func) + async def inner(*args, **kwds): + async with self._recreate_cm(): + return await func(*args, **kwds) + return inner + + class _GeneratorContextManagerBase: """Shared functionality for @contextmanager and @asynccontextmanager.""" @@ -167,9 +183,16 @@ class _GeneratorContextManager(_GeneratorContextManagerBase, class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, - AbstractAsyncContextManager): + AbstractAsyncContextManager, + AsyncContextDecorator): """Helper for @asynccontextmanager.""" + def _recreate_cm(self): + # _AGCM instances are one-shot context managers, so the + # ACM must be recreated each time a decorated function is + # called + return self.__class__(self.func, self.args, self.kwds) + async def __aenter__(self): try: return await self.gen.__anext__() diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 3765f6cbf28..109807d633d 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -278,6 +278,33 @@ class AsyncContextManagerTestCase(unittest.TestCase): async with woohoo(self=11, func=22, args=33, kwds=44) as target: self.assertEqual(target, (11, 22, 33, 44)) + @_async_test + async def test_recursive(self): + depth = 0 + ncols = 0 + + @asynccontextmanager + async def woohoo(): + nonlocal ncols + ncols += 1 + + nonlocal depth + before = depth + depth += 1 + yield + depth -= 1 + self.assertEqual(depth, before) + + @woohoo() + async def recursive(): + if depth < 10: + await recursive() + + await recursive() + + self.assertEqual(ncols, 10) + self.assertEqual(depth, 0) + class AclosingTestCase(unittest.TestCase): diff --git a/Misc/NEWS.d/next/Library/2020-05-29-15-25-41.bpo-40816.w61Pob.rst b/Misc/NEWS.d/next/Library/2020-05-29-15-25-41.bpo-40816.w61Pob.rst new file mode 100644 index 00000000000..66b75779784 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-05-29-15-25-41.bpo-40816.w61Pob.rst @@ -0,0 +1 @@ +Add AsyncContextDecorator to contextlib to support async context manager as a decorator. \ No newline at end of file From 80449f243b13311d660eab3a751648029bcdd833 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Thu, 5 Nov 2020 09:23:15 +0000 Subject: [PATCH 31/61] bpo-42266: Handle monkey-patching descriptors in LOAD_ATTR cache (GH-23157) --- Lib/test/test_opcache.py | 23 +++++++++++++++++++ .../2020-11-04-23-03-25.bpo-42266.G4hGDe.rst | 3 +++ PCbuild/lib.pyproj | 1 + Python/ceval.c | 8 +------ 4 files changed, 28 insertions(+), 7 deletions(-) create mode 100644 Lib/test/test_opcache.py create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-04-23-03-25.bpo-42266.G4hGDe.rst diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py new file mode 100644 index 00000000000..61f337d70ea --- /dev/null +++ b/Lib/test/test_opcache.py @@ -0,0 +1,23 @@ +import unittest + +class TestLoadAttrCache(unittest.TestCase): + def test_descriptor_added_after_optimization(self): + class Descriptor: + pass + + class C: + def __init__(self): + self.x = 1 + x = Descriptor() + + def f(o): + return o.x + + o = C() + for i in range(1025): + assert f(o) == 1 + + Descriptor.__get__ = lambda self, instance, value: 2 + Descriptor.__set__ = lambda *args: None + + self.assertEqual(f(o), 2) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-04-23-03-25.bpo-42266.G4hGDe.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-04-23-03-25.bpo-42266.G4hGDe.rst new file mode 100644 index 00000000000..a8598cfde04 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-04-23-03-25.bpo-42266.G4hGDe.rst @@ -0,0 +1,3 @@ +Fixed a bug with the LOAD_ATTR opcode cache that was not respecting +monkey-patching a class-level attribute to make it a descriptor. Patch by +Pablo Galindo. diff --git a/PCbuild/lib.pyproj b/PCbuild/lib.pyproj index a15165d92ef..1be60b1a11b 100644 --- a/PCbuild/lib.pyproj +++ b/PCbuild/lib.pyproj @@ -1196,6 +1196,7 @@ + diff --git a/Python/ceval.c b/Python/ceval.c index 13b209fc706..32e3019682f 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -3179,7 +3179,6 @@ main_loop: if (co_opcache != NULL && /* co_opcache can be NULL after a DEOPT() call. */ type->tp_getattro == PyObject_GenericGetAttr) { - PyObject *descr; Py_ssize_t ret; if (type->tp_dictoffset > 0) { @@ -3190,12 +3189,7 @@ main_loop: goto error; } } - - descr = _PyType_Lookup(type, name); - if (descr == NULL || - Py_TYPE(descr)->tp_descr_get == NULL || - !PyDescr_IsData(descr)) - { + if (_PyType_Lookup(type, name) == NULL) { dictptr = (PyObject **) ((char *)owner + type->tp_dictoffset); dict = *dictptr; From 53a03aafd5812018a3821a2e83063fd3d6cd2576 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2020 15:02:12 +0100 Subject: [PATCH 32/61] bpo-42262: Add Py_NewRef() and Py_XNewRef() (GH-23152) Added Py_NewRef() and Py_XNewRef() functions to increment the reference count of an object and return the object. --- Doc/c-api/refcounting.rst | 31 +++++++++++++++++++ Doc/whatsnew/3.10.rst | 4 +++ Include/boolobject.h | 4 +-- Include/object.h | 30 ++++++++++++++++-- .../2020-11-04-17-22-36.bpo-42262.fCWzBb.rst | 2 ++ Objects/object.c | 16 ++++++++++ PC/python3dll.c | 4 ++- 7 files changed, 85 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-04-17-22-36.bpo-42262.fCWzBb.rst diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index 0df12c67f40..b15c0e6aecc 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -16,12 +16,43 @@ objects. Increment the reference count for object *o*. The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + See also :c:func:`Py_NewRef`. + .. c:function:: void Py_XINCREF(PyObject *o) Increment the reference count for object *o*. The object may be ``NULL``, in which case the macro has no effect. + See also :c:func:`Py_XNewRef`. + + +.. c:function:: PyObject* Py_NewRef(PyObject *o) + + Increment the reference count of the object *o* and return the object *o*. + + The object *o* must not be ``NULL``. + + For example:: + + Py_INCREF(obj); + self->attr = obj; + + can be written as:: + + self->attr = Py_NewRef(obj); + + .. versionadded:: 3.10 + + +.. c:function:: PyObject* Py_XNewRef(PyObject *o) + + Similar to :c:func:`Py_NewRef`, but the object *o* can be NULL. + + If the object *o* is ``NULL``, the function just returns ``NULL``. + + .. versionadded:: 3.10 + .. c:function:: void Py_DECREF(PyObject *o) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 9d9284897be..bac1a2e6783 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -379,6 +379,10 @@ New Features success. (Contributed by Victor Stinner in :issue:`1635741`.) +* Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the + reference count of an object and return the object. + (Contributed by Victor Stinner in :issue:`42262`.) + Porting to Python 3.10 ---------------------- diff --git a/Include/boolobject.h b/Include/boolobject.h index bb8044a2b02..6673d7206c0 100644 --- a/Include/boolobject.h +++ b/Include/boolobject.h @@ -22,8 +22,8 @@ PyAPI_DATA(struct _longobject) _Py_FalseStruct, _Py_TrueStruct; #define Py_True ((PyObject *) &_Py_TrueStruct) /* Macros for returning Py_True or Py_False, respectively */ -#define Py_RETURN_TRUE return Py_INCREF(Py_True), Py_True -#define Py_RETURN_FALSE return Py_INCREF(Py_False), Py_False +#define Py_RETURN_TRUE return Py_NewRef(Py_True) +#define Py_RETURN_FALSE return Py_NewRef(Py_False) /* Function to return a bool from a C long */ PyAPI_FUNC(PyObject *) PyBool_FromLong(long); diff --git a/Include/object.h b/Include/object.h index 6ee4ee78485..835d9de01fb 100644 --- a/Include/object.h +++ b/Include/object.h @@ -526,6 +526,31 @@ they can have object code that is not dependent on Python compilation flags. PyAPI_FUNC(void) Py_IncRef(PyObject *); PyAPI_FUNC(void) Py_DecRef(PyObject *); +// Increment the reference count of the object and return the object. +PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); + +// Similar to Py_NewRef() but the object can be NULL. +PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); + +static inline PyObject* _Py_NewRef(PyObject *obj) +{ + Py_INCREF(obj); + return obj; +} + +static inline PyObject* _Py_XNewRef(PyObject *obj) +{ + Py_XINCREF(obj); + return obj; +} + +// Py_NewRef() and Py_XNewRef() are exported as functions for the stable ABI. +// Names overriden with macros by static inline functions for best +// performances. +#define Py_NewRef(obj) _Py_NewRef(obj) +#define Py_XNewRef(obj) _Py_XNewRef(obj) + + /* _Py_NoneStruct is an object of undefined type which can be used in contexts where NULL (nil) is not suitable (since NULL often means 'error'). @@ -536,7 +561,7 @@ PyAPI_DATA(PyObject) _Py_NoneStruct; /* Don't use this directly */ #define Py_None (&_Py_NoneStruct) /* Macro for returning Py_None from a function */ -#define Py_RETURN_NONE return Py_INCREF(Py_None), Py_None +#define Py_RETURN_NONE return Py_NewRef(Py_None) /* Py_NotImplemented is a singleton used to signal that an operation is @@ -546,8 +571,7 @@ PyAPI_DATA(PyObject) _Py_NotImplementedStruct; /* Don't use this directly */ #define Py_NotImplemented (&_Py_NotImplementedStruct) /* Macro for returning Py_NotImplemented from a function */ -#define Py_RETURN_NOTIMPLEMENTED \ - return Py_INCREF(Py_NotImplemented), Py_NotImplemented +#define Py_RETURN_NOTIMPLEMENTED return Py_NewRef(Py_NotImplemented) /* Rich comparison opcodes */ #define Py_LT 0 diff --git a/Misc/NEWS.d/next/C API/2020-11-04-17-22-36.bpo-42262.fCWzBb.rst b/Misc/NEWS.d/next/C API/2020-11-04-17-22-36.bpo-42262.fCWzBb.rst new file mode 100644 index 00000000000..8c1e4f41844 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-04-17-22-36.bpo-42262.fCWzBb.rst @@ -0,0 +1,2 @@ +Added :c:func:`Py_NewRef` and :c:func:`Py_XNewRef` functions to increment the +reference count of an object and return the object. Patch by Victor Stinner. diff --git a/Objects/object.c b/Objects/object.c index 7bc3e48d40a..be7790eefd1 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2208,6 +2208,22 @@ PyObject_GET_WEAKREFS_LISTPTR(PyObject *op) } +#undef Py_NewRef +#undef Py_XNewRef + +// Export Py_NewRef() and Py_XNewRef() as regular functions for the stable ABI. +PyObject* +Py_NewRef(PyObject *obj) +{ + return _Py_NewRef(obj); +} + +PyObject* +Py_XNewRef(PyObject *obj) +{ + return _Py_XNewRef(obj); +} + #ifdef __cplusplus } #endif diff --git a/PC/python3dll.c b/PC/python3dll.c index 7e4a5101773..d1fdd0ac54c 100644 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -40,8 +40,8 @@ EXPORT_FUNC(Py_AddPendingCall) EXPORT_FUNC(Py_AtExit) EXPORT_FUNC(Py_BuildValue) EXPORT_FUNC(Py_CompileString) -EXPORT_FUNC(Py_DecodeLocale) EXPORT_FUNC(Py_DecRef) +EXPORT_FUNC(Py_DecodeLocale) EXPORT_FUNC(Py_EncodeLocale) EXPORT_FUNC(Py_EndInterpreter) EXPORT_FUNC(Py_EnterRecursiveCall) @@ -72,6 +72,7 @@ EXPORT_FUNC(Py_LeaveRecursiveCall) EXPORT_FUNC(Py_Main) EXPORT_FUNC(Py_MakePendingCalls) EXPORT_FUNC(Py_NewInterpreter) +EXPORT_FUNC(Py_NewRef) EXPORT_FUNC(Py_ReprEnter) EXPORT_FUNC(Py_ReprLeave) EXPORT_FUNC(Py_SetPath) @@ -80,6 +81,7 @@ EXPORT_FUNC(Py_SetPythonHome) EXPORT_FUNC(Py_SetRecursionLimit) EXPORT_FUNC(Py_SymtableString) EXPORT_FUNC(Py_VaBuildValue) +EXPORT_FUNC(Py_XNewRef) EXPORT_FUNC(PyArg_Parse) EXPORT_FUNC(PyArg_ParseTuple) EXPORT_FUNC(PyArg_ParseTupleAndKeywords) From 133aa2d5816b69d8ee755e1a9d2d1977b9205736 Mon Sep 17 00:00:00 2001 From: kj <28750310+Fidget-Spinner@users.noreply.github.com> Date: Fri, 6 Nov 2020 00:16:27 +0800 Subject: [PATCH 33/61] [docs] fix wrongly named AsyncContextDecorator (GH-23164) Also added versionchanged. --- Doc/library/contextlib.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index ee2becb8dff..d5a1068a734 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -409,11 +409,11 @@ Functions and classes provided: .. versionadded:: 3.2 -.. class:: AsyncContextManager +.. class:: AsyncContextDecorator - Similar as ContextManger only for async + Similar to :class:`ContextDecorator` but only for asynchronous functions. - Example of ``ContextDecorator``:: + Example of ``AsyncContextDecorator``:: from asyncio import run from contextlib import AsyncContextDecorator @@ -445,6 +445,8 @@ Functions and classes provided: The bit in the middle Finishing + .. versionadded:: 3.10 + .. class:: ExitStack() From 4662fa9bfe4a849fe87bfb321d8ef0956c89a772 Mon Sep 17 00:00:00 2001 From: vabr-g Date: Thu, 5 Nov 2020 18:04:38 +0100 Subject: [PATCH 34/61] bpo-41877 Check for asert, aseert, assrt in mocks (GH-23165) Currently, a Mock object which is not unsafe will raise an AttributeError if an attribute with the prefix assert or assret is accessed on it. This protects against misspellings of real assert method calls, which lead to tests passing silently even if the tested code does not satisfy the intended assertion. Recently a check was done in a large code base (Google) and three more frequent ways of misspelling assert were found causing harm: asert, aseert, assrt. These are now added to the existing check. --- Lib/unittest/mock.py | 4 ++-- Lib/unittest/test/testmock/testmock.py | 11 ++++++++++- .../Library/2020-11-05-16-00-03.bpo-41877.FHbngM.rst | 2 ++ 3 files changed, 14 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-05-16-00-03.bpo-41877.FHbngM.rst diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index b495a5f6ccc..f5f502f2572 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -631,9 +631,9 @@ class NonCallableMock(Base): elif _is_magic(name): raise AttributeError(name) if not self._mock_unsafe: - if name.startswith(('assert', 'assret')): + if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')): raise AttributeError("Attributes cannot start with 'assert' " - "or 'assret'") + "or its misspellings") result = self._mock_children.get(name) if result is _deleted: diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/unittest/test/testmock/testmock.py index ce674e713e9..194ce3f61bb 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/unittest/test/testmock/testmock.py @@ -1598,14 +1598,23 @@ class MockTest(unittest.TestCase): #Issue21238 def test_mock_unsafe(self): m = Mock() - msg = "Attributes cannot start with 'assert' or 'assret'" + msg = "Attributes cannot start with 'assert' or its misspellings" with self.assertRaisesRegex(AttributeError, msg): m.assert_foo_call() with self.assertRaisesRegex(AttributeError, msg): m.assret_foo_call() + with self.assertRaisesRegex(AttributeError, msg): + m.asert_foo_call() + with self.assertRaisesRegex(AttributeError, msg): + m.aseert_foo_call() + with self.assertRaisesRegex(AttributeError, msg): + m.assrt_foo_call() m = Mock(unsafe=True) m.assert_foo_call() m.assret_foo_call() + m.asert_foo_call() + m.aseert_foo_call() + m.assrt_foo_call() #Issue21262 def test_assert_not_called(self): diff --git a/Misc/NEWS.d/next/Library/2020-11-05-16-00-03.bpo-41877.FHbngM.rst b/Misc/NEWS.d/next/Library/2020-11-05-16-00-03.bpo-41877.FHbngM.rst new file mode 100644 index 00000000000..6f6fccb1d4c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-05-16-00-03.bpo-41877.FHbngM.rst @@ -0,0 +1,2 @@ +Mock objects which are not unsafe will now raise an AttributeError if an attribute with the prefix asert, aseert, +or assrt is accessed, in addition to this already happening for the prefixes assert or assret. \ No newline at end of file From f3cb81431574453aac3b6dcadb3120331e6a8f1c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2020 18:12:33 +0100 Subject: [PATCH 35/61] bpo-42260: Add _PyConfig_FromDict() (GH-23167) * Rename config_as_dict() to _PyConfig_AsDict(). * Add 'module_search_paths_set' to _PyConfig_AsDict(). * Add _PyConfig_FromDict(). * Add get_config() and set_config() to _testinternalcapi. * Add config_check_consistency(). --- Include/internal/pycore_initconfig.h | 3 + Lib/test/_test_embed_set_config.py | 243 ++++++++++++++++ Lib/test/test_embed.py | 14 + Modules/_testinternalcapi.c | 39 ++- Python/initconfig.c | 410 +++++++++++++++++++++++---- Python/sysmodule.c | 4 +- 6 files changed, 653 insertions(+), 60 deletions(-) create mode 100644 Lib/test/_test_embed_set_config.py diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index df7ad779f47..325be5494d4 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -158,6 +158,9 @@ extern PyStatus _PyConfig_SetPyArgv( PyConfig *config, const _PyArgv *args); +PyAPI_FUNC(PyObject*) _PyConfig_AsDict(const PyConfig *config); +PyAPI_FUNC(int) _PyConfig_FromDict(PyConfig *config, PyObject *dict); + /* --- Function used for testing ---------------------------------- */ diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py new file mode 100644 index 00000000000..7c913811ded --- /dev/null +++ b/Lib/test/_test_embed_set_config.py @@ -0,0 +1,243 @@ +# bpo-42260: Test _PyInterpreterState_GetConfigCopy() +# and _PyInterpreterState_SetConfig(). +# +# Test run in a subinterpreter since set_config(get_config()) +# does reset sys attributes to their state of the Python startup +# (before the site module is run). + +import _testinternalcapi +import os +import sys +import unittest + + +MS_WINDOWS = (os.name == 'nt') +MAX_HASH_SEED = 4294967295 + +class SetConfigTests(unittest.TestCase): + def setUp(self): + self.old_config = _testinternalcapi.get_config() + self.sys_copy = dict(sys.__dict__) + + def tearDown(self): + self.set_config(parse_argv=0) + sys.__dict__.clear() + sys.__dict__.update(self.sys_copy) + + def set_config(self, **kwargs): + _testinternalcapi.set_config(self.old_config | kwargs) + + def check(self, **kwargs): + self.set_config(**kwargs) + for key, value in kwargs.items(): + self.assertEqual(getattr(sys, key), value, + (key, value)) + + def test_set_invalid(self): + invalid_uint = -1 + NULL = None + invalid_wstr = NULL + # PyWideStringList strings must be non-NULL + invalid_wstrlist = ["abc", NULL, "def"] + + type_tests = [] + value_tests = [ + # enum + ('_config_init', 0), + ('_config_init', 4), + # unsigned long + ("hash_seed", -1), + ("hash_seed", MAX_HASH_SEED + 1), + ] + + # int (unsigned) + options = [ + '_config_init', + 'isolated', + 'use_environment', + 'dev_mode', + 'install_signal_handlers', + 'use_hash_seed', + 'faulthandler', + 'tracemalloc', + 'import_time', + 'show_ref_count', + 'dump_refs', + 'malloc_stats', + 'parse_argv', + 'site_import', + 'bytes_warning', + 'inspect', + 'interactive', + 'optimization_level', + 'parser_debug', + 'write_bytecode', + 'verbose', + 'quiet', + 'user_site_directory', + 'configure_c_stdio', + 'buffered_stdio', + 'pathconfig_warnings', + 'module_search_paths_set', + 'skip_source_first_line', + '_install_importlib', + '_init_main', + '_isolated_interpreter', + ] + if MS_WINDOWS: + options.append('legacy_windows_stdio') + for key in options: + value_tests.append((key, invalid_uint)) + type_tests.append((key, "abc")) + type_tests.append((key, 2.0)) + + # wchar_t* + for key in ( + 'filesystem_encoding', + 'filesystem_errors', + 'stdio_encoding', + 'stdio_errors', + 'check_hash_pycs_mode', + 'program_name', + 'platlibdir', + 'executable', + 'base_executable', + 'prefix', + 'base_prefix', + 'exec_prefix', + 'base_exec_prefix', + # optional wstr: + # 'pythonpath_env' + # 'home', + # 'pycache_prefix' + # 'run_command' + # 'run_module' + # 'run_filename' + ): + value_tests.append((key, invalid_wstr)) + type_tests.append((key, b'bytes')) + type_tests.append((key, 123)) + + # PyWideStringList + for key in ( + 'orig_argv', + 'argv', + 'xoptions', + 'warnoptions', + 'module_search_paths', + ): + value_tests.append((key, invalid_wstrlist)) + type_tests.append((key, 123)) + type_tests.append((key, "abc")) + type_tests.append((key, [123])) + type_tests.append((key, [b"bytes"])) + + + if MS_WINDOWS: + value_tests.append(('legacy_windows_stdio', invalid_uint)) + + for exc_type, tests in ( + (ValueError, value_tests), + (TypeError, type_tests), + ): + for key, value in tests: + config = self.old_config | {key: value} + with self.subTest(key=key, value=value, exc_type=exc_type): + with self.assertRaises(exc_type): + _testinternalcapi.set_config(config) + + def test_flags(self): + for sys_attr, key, value in ( + ("debug", "parser_debug", 1), + ("inspect", "inspect", 2), + ("interactive", "interactive", 3), + ("optimize", "optimization_level", 4), + ("verbose", "verbose", 1), + ("bytes_warning", "bytes_warning", 10), + ("quiet", "quiet", 11), + ("isolated", "isolated", 12), + ): + with self.subTest(sys=sys_attr, key=key, value=value): + self.set_config(**{key: value, 'parse_argv': 0}) + self.assertEqual(getattr(sys.flags, sys_attr), value) + + self.set_config(write_bytecode=0) + self.assertEqual(sys.flags.dont_write_bytecode, True) + self.assertEqual(sys.dont_write_bytecode, True) + + self.set_config(write_bytecode=1) + self.assertEqual(sys.flags.dont_write_bytecode, False) + self.assertEqual(sys.dont_write_bytecode, False) + + self.set_config(user_site_directory=0, isolated=0) + self.assertEqual(sys.flags.no_user_site, 1) + self.set_config(user_site_directory=1, isolated=0) + self.assertEqual(sys.flags.no_user_site, 0) + + self.set_config(site_import=0) + self.assertEqual(sys.flags.no_site, 1) + self.set_config(site_import=1) + self.assertEqual(sys.flags.no_site, 0) + + self.set_config(dev_mode=0) + self.assertEqual(sys.flags.dev_mode, False) + self.set_config(dev_mode=1) + self.assertEqual(sys.flags.dev_mode, True) + + self.set_config(use_environment=0, isolated=0) + self.assertEqual(sys.flags.ignore_environment, 1) + self.set_config(use_environment=1, isolated=0) + self.assertEqual(sys.flags.ignore_environment, 0) + + self.set_config(use_hash_seed=1, hash_seed=0) + self.assertEqual(sys.flags.hash_randomization, 0) + self.set_config(use_hash_seed=0, hash_seed=0) + self.assertEqual(sys.flags.hash_randomization, 1) + self.set_config(use_hash_seed=1, hash_seed=123) + self.assertEqual(sys.flags.hash_randomization, 1) + + def test_options(self): + self.check(warnoptions=[]) + self.check(warnoptions=["default", "ignore"]) + + self.set_config(xoptions=[]) + self.assertEqual(sys._xoptions, {}) + self.set_config(xoptions=["dev", "tracemalloc=5"]) + self.assertEqual(sys._xoptions, {"dev": True, "tracemalloc": "5"}) + + def test_pathconfig(self): + self.check( + executable='executable', + prefix="prefix", + base_prefix="base_prefix", + exec_prefix="exec_prefix", + base_exec_prefix="base_exec_prefix", + platlibdir="platlibdir") + + self.set_config(base_executable="base_executable") + self.assertEqual(sys._base_executable, "base_executable") + + def test_path(self): + self.set_config(module_search_paths_set=1, + module_search_paths=['a', 'b', 'c']) + self.assertEqual(sys.path, ['a', 'b', 'c']) + + # Leave sys.path unchanged if module_search_paths_set=0 + self.set_config(module_search_paths_set=0, + module_search_paths=['new_path']) + self.assertEqual(sys.path, ['a', 'b', 'c']) + + def test_argv(self): + self.set_config(parse_argv=0, + argv=['python_program', 'args'], + orig_argv=['orig', 'orig_args']) + self.assertEqual(sys.argv, ['python_program', 'args']) + self.assertEqual(sys.orig_argv, ['orig', 'orig_args']) + + def test_pycache_prefix(self): + self.check(pycache_prefix=None) + self.check(pycache_prefix="pycache_prefix") + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 36a0e77e14c..91820615193 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -30,6 +30,8 @@ API_PYTHON = 2 # _PyCoreConfig_InitIsolatedConfig() API_ISOLATED = 3 +MAX_HASH_SEED = 4294967295 + def debug_build(program): program = os.path.basename(program) @@ -382,6 +384,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'exec_prefix': GET_DEFAULT_CONFIG, 'base_exec_prefix': GET_DEFAULT_CONFIG, 'module_search_paths': GET_DEFAULT_CONFIG, + 'module_search_paths_set': 1, 'platlibdir': sys.platlibdir, 'site_import': 1, @@ -1408,6 +1411,17 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): # ignore output +class SetConfigTests(unittest.TestCase): + def test_set_config(self): + # bpo-42260: Test _PyInterpreterState_SetConfig() + cmd = [sys.executable, '-I', '-m', 'test._test_embed_set_config'] + proc = subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertEqual(proc.returncode, 0, + (proc.returncode, proc.stdout, proc.stderr)) + + class AuditingTests(EmbeddingTestsMixin, unittest.TestCase): def test_open_code_hook(self): self.run_embedded_interpreter("test_open_code_hook") diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index ad74af8363e..be144bfba02 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -13,9 +13,10 @@ #include "Python.h" #include "pycore_bitutils.h" // _Py_bswap32() -#include "pycore_initconfig.h" // _Py_GetConfigsAsDict() -#include "pycore_hashtable.h" // _Py_hashtable_new() #include "pycore_gc.h" // PyGC_Head +#include "pycore_hashtable.h" // _Py_hashtable_new() +#include "pycore_initconfig.h" // _Py_GetConfigsAsDict() +#include "pycore_interp.h" // _PyInterpreterState_GetConfigCopy() static PyObject * @@ -231,6 +232,38 @@ test_hashtable(PyObject *self, PyObject *Py_UNUSED(args)) } +static PyObject * +test_get_config(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(args)) +{ + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + if (_PyInterpreterState_GetConfigCopy(&config) < 0) { + PyConfig_Clear(&config); + return NULL; + } + PyObject *dict = _PyConfig_AsDict(&config); + PyConfig_Clear(&config); + return dict; +} + + +static PyObject * +test_set_config(PyObject *Py_UNUSED(self), PyObject *dict) +{ + PyConfig config; + PyConfig_InitIsolatedConfig(&config); + if (_PyConfig_FromDict(&config, dict) < 0) { + PyConfig_Clear(&config); + return NULL; + } + if (_PyInterpreterState_SetConfig(&config) < 0) { + return NULL; + } + PyConfig_Clear(&config); + Py_RETURN_NONE; +} + + static PyMethodDef TestMethods[] = { {"get_configs", get_configs, METH_NOARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, @@ -238,6 +271,8 @@ static PyMethodDef TestMethods[] = { {"test_popcount", test_popcount, METH_NOARGS}, {"test_bit_length", test_bit_length, METH_NOARGS}, {"test_hashtable", test_hashtable, METH_NOARGS}, + {"get_config", test_get_config, METH_NOARGS}, + {"set_config", test_set_config, METH_O}, {NULL, NULL} /* sentinel */ }; diff --git a/Python/initconfig.c b/Python/initconfig.c index de496ac7b52..d54d5b7a999 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -577,6 +577,74 @@ Py_GetArgcArgv(int *argc, wchar_t ***argv) ? _PyStatus_ERR("cannot decode " NAME) \ : _PyStatus_NO_MEMORY()) +#define MAX_HASH_SEED 4294967295UL + + +#ifndef NDEBUG +static int +config_check_consistency(const PyConfig *config) +{ + /* Check config consistency */ + assert(config->isolated >= 0); + assert(config->use_environment >= 0); + assert(config->dev_mode >= 0); + assert(config->install_signal_handlers >= 0); + assert(config->use_hash_seed >= 0); + assert(config->hash_seed <= MAX_HASH_SEED); + assert(config->faulthandler >= 0); + assert(config->tracemalloc >= 0); + assert(config->import_time >= 0); + assert(config->show_ref_count >= 0); + assert(config->dump_refs >= 0); + assert(config->malloc_stats >= 0); + assert(config->site_import >= 0); + assert(config->bytes_warning >= 0); + assert(config->inspect >= 0); + assert(config->interactive >= 0); + assert(config->optimization_level >= 0); + assert(config->parser_debug >= 0); + assert(config->write_bytecode >= 0); + assert(config->verbose >= 0); + assert(config->quiet >= 0); + assert(config->user_site_directory >= 0); + assert(config->parse_argv >= 0); + assert(config->configure_c_stdio >= 0); + assert(config->buffered_stdio >= 0); + assert(config->program_name != NULL); + assert(_PyWideStringList_CheckConsistency(&config->orig_argv)); + assert(_PyWideStringList_CheckConsistency(&config->argv)); + /* sys.argv must be non-empty: empty argv is replaced with [''] */ + assert(config->argv.length >= 1); + assert(_PyWideStringList_CheckConsistency(&config->xoptions)); + assert(_PyWideStringList_CheckConsistency(&config->warnoptions)); + assert(_PyWideStringList_CheckConsistency(&config->module_search_paths)); + assert(config->module_search_paths_set >= 0); + if (config->_install_importlib) { + /* don't check config->module_search_paths */ + assert(config->executable != NULL); + assert(config->base_executable != NULL); + assert(config->prefix != NULL); + assert(config->base_prefix != NULL); + assert(config->exec_prefix != NULL); + assert(config->base_exec_prefix != NULL); + } + assert(config->platlibdir != NULL); + assert(config->filesystem_encoding != NULL); + assert(config->filesystem_errors != NULL); + assert(config->stdio_encoding != NULL); + assert(config->stdio_errors != NULL); +#ifdef MS_WINDOWS + assert(config->legacy_windows_stdio >= 0); +#endif + /* -c and -m options are exclusive */ + assert(!(config->run_command != NULL && config->run_module != NULL)); + assert(config->check_hash_pycs_mode != NULL); + assert(config->_install_importlib >= 0); + assert(config->pathconfig_warnings >= 0); + return 1; +} +#endif + /* Free memory allocated in config, but don't clear all attributes */ void @@ -880,8 +948,8 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2) } -static PyObject * -config_as_dict(const PyConfig *config) +PyObject * +_PyConfig_AsDict(const PyConfig *config) { PyObject *dict = PyDict_New(); if (dict == NULL) { @@ -936,6 +1004,7 @@ config_as_dict(const PyConfig *config) SET_ITEM_WSTRLIST(warnoptions); SET_ITEM_WSTR(pythonpath_env); SET_ITEM_WSTR(home); + SET_ITEM_INT(module_search_paths_set); SET_ITEM_WSTRLIST(module_search_paths); SET_ITEM_WSTR(executable); SET_ITEM_WSTR(base_executable); @@ -987,6 +1056,285 @@ fail: } +static PyObject* +config_dict_get(PyObject *dict, const char *name) +{ + PyObject *item = PyDict_GetItemString(dict, name); + if (item == NULL) { + PyErr_Format(PyExc_ValueError, "missing config key: %s", name); + return NULL; + } + return item; +} + + +static void +config_dict_invalid_value(const char *name) +{ + PyErr_Format(PyExc_ValueError, "invalid config value: %s", name); +} + + +static void +config_dict_invalid_type(const char *name) +{ + PyErr_Format(PyExc_TypeError, "invalid config type: %s", name); +} + + +static int +config_dict_get_int(PyObject *dict, const char *name, int *result) +{ + PyObject *item = config_dict_get(dict, name); + if (item == NULL) { + return -1; + } + int value = _PyLong_AsInt(item); + if (value == -1 && PyErr_Occurred()) { + if (PyErr_ExceptionMatches(PyExc_TypeError)) { + config_dict_invalid_type(name); + } + else { + config_dict_invalid_value(name); + } + return -1; + } + *result = value; + return 0; +} + + +static int +config_dict_get_ulong(PyObject *dict, const char *name, unsigned long *result) +{ + PyObject *item = config_dict_get(dict, name); + if (item == NULL) { + return -1; + } + unsigned long value = PyLong_AsUnsignedLong(item); + if (value == (unsigned long)-1 && PyErr_Occurred()) { + config_dict_invalid_value(name); + return -1; + } + *result = value; + return 0; +} + + +static int +config_dict_get_wstr(PyObject *dict, const char *name, PyConfig *config, + wchar_t **result) +{ + PyObject *item = config_dict_get(dict, name); + if (item == NULL) { + return -1; + } + PyStatus status; + if (item == Py_None) { + status = PyConfig_SetString(config, result, NULL); + } + else if (!PyUnicode_Check(item)) { + config_dict_invalid_type(name); + return -1; + } + else { + wchar_t *wstr = PyUnicode_AsWideCharString(item, NULL); + if (wstr == NULL) { + return -1; + } + status = PyConfig_SetString(config, result, wstr); + PyMem_Free(wstr); + } + if (_PyStatus_EXCEPTION(status)) { + PyErr_NoMemory(); + return -1; + } + return 0; +} + + +static int +config_dict_get_wstrlist(PyObject *dict, const char *name, PyConfig *config, + PyWideStringList *result) +{ + PyObject *list = config_dict_get(dict, name); + if (list == NULL) { + return -1; + } + + if (!PyList_CheckExact(list)) { + config_dict_invalid_type(name); + return -1; + } + + PyWideStringList wstrlist = _PyWideStringList_INIT; + for (Py_ssize_t i=0; i < PyList_GET_SIZE(list); i++) { + PyObject *item = PyList_GET_ITEM(list, i); + + if (item == Py_None) { + config_dict_invalid_value(name); + goto error; + } + else if (!PyUnicode_Check(item)) { + config_dict_invalid_type(name); + goto error; + } + wchar_t *wstr = PyUnicode_AsWideCharString(item, NULL); + if (wstr == NULL) { + goto error; + } + PyStatus status = PyWideStringList_Append(&wstrlist, wstr); + PyMem_Free(wstr); + if (_PyStatus_EXCEPTION(status)) { + PyErr_NoMemory(); + goto error; + } + } + + if (_PyWideStringList_Copy(result, &wstrlist) < 0) { + PyErr_NoMemory(); + goto error; + } + _PyWideStringList_Clear(&wstrlist); + return 0; + +error: + _PyWideStringList_Clear(&wstrlist); + return -1; +} + + +int +_PyConfig_FromDict(PyConfig *config, PyObject *dict) +{ + if (!PyDict_Check(dict)) { + PyErr_SetString(PyExc_TypeError, "dict expected"); + return -1; + } + +#define CHECK_VALUE(NAME, TEST) \ + if (!(TEST)) { \ + config_dict_invalid_value(NAME); \ + return -1; \ + } +#define GET_UINT(KEY) \ + do { \ + if (config_dict_get_int(dict, #KEY, &config->KEY) < 0) { \ + return -1; \ + } \ + CHECK_VALUE(#KEY, config->KEY >= 0); \ + } while (0) +#define GET_WSTR(KEY) \ + do { \ + if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ + return -1; \ + } \ + CHECK_VALUE(#KEY, config->KEY != NULL); \ + } while (0) +#define GET_WSTR_OPT(KEY) \ + do { \ + if (config_dict_get_wstr(dict, #KEY, config, &config->KEY) < 0) { \ + return -1; \ + } \ + } while (0) +#define GET_WSTRLIST(KEY) \ + do { \ + if (config_dict_get_wstrlist(dict, #KEY, config, &config->KEY) < 0) { \ + return -1; \ + } \ + } while (0) + + GET_UINT(_config_init); + CHECK_VALUE("_config_init", + config->_config_init == _PyConfig_INIT_COMPAT + || config->_config_init == _PyConfig_INIT_PYTHON + || config->_config_init == _PyConfig_INIT_ISOLATED); + GET_UINT(isolated); + GET_UINT(use_environment); + GET_UINT(dev_mode); + GET_UINT(install_signal_handlers); + GET_UINT(use_hash_seed); + if (config_dict_get_ulong(dict, "hash_seed", &config->hash_seed) < 0) { + return -1; + } + CHECK_VALUE("hash_seed", config->hash_seed <= MAX_HASH_SEED); + GET_UINT(faulthandler); + GET_UINT(tracemalloc); + GET_UINT(import_time); + GET_UINT(show_ref_count); + GET_UINT(dump_refs); + GET_UINT(malloc_stats); + GET_WSTR(filesystem_encoding); + GET_WSTR(filesystem_errors); + GET_WSTR_OPT(pycache_prefix); + GET_UINT(parse_argv); + GET_WSTRLIST(orig_argv); + GET_WSTRLIST(argv); + GET_WSTRLIST(xoptions); + GET_WSTRLIST(warnoptions); + GET_UINT(site_import); + GET_UINT(bytes_warning); + GET_UINT(inspect); + GET_UINT(interactive); + GET_UINT(optimization_level); + GET_UINT(parser_debug); + GET_UINT(write_bytecode); + GET_UINT(verbose); + GET_UINT(quiet); + GET_UINT(user_site_directory); + GET_UINT(configure_c_stdio); + GET_UINT(buffered_stdio); + GET_WSTR(stdio_encoding); + GET_WSTR(stdio_errors); +#ifdef MS_WINDOWS + GET_UINT(legacy_windows_stdio); +#endif + GET_WSTR(check_hash_pycs_mode); + + GET_UINT(pathconfig_warnings); + GET_WSTR(program_name); + GET_WSTR_OPT(pythonpath_env); + GET_WSTR_OPT(home); + GET_WSTR(platlibdir); + + GET_UINT(module_search_paths_set); + GET_WSTRLIST(module_search_paths); + if (config->_install_importlib) { + GET_WSTR(executable); + GET_WSTR(base_executable); + GET_WSTR(prefix); + GET_WSTR(base_prefix); + GET_WSTR(exec_prefix); + GET_WSTR(base_exec_prefix); + } + else { + GET_WSTR_OPT(executable); + GET_WSTR_OPT(base_executable); + GET_WSTR_OPT(prefix); + GET_WSTR_OPT(base_prefix); + GET_WSTR_OPT(exec_prefix); + GET_WSTR_OPT(base_exec_prefix); + } + + GET_UINT(skip_source_first_line); + GET_WSTR_OPT(run_command); + GET_WSTR_OPT(run_module); + GET_WSTR_OPT(run_filename); + + GET_UINT(_install_importlib); + GET_UINT(_init_main); + GET_UINT(_isolated_interpreter); + + assert(config_check_consistency(config)); + +#undef CHECK_VALUE +#undef GET_UINT +#undef GET_WSTR +#undef GET_WSTR_OPT + return 0; +} + + static const char* config_get_env(const PyConfig *config, const char *name) { @@ -1254,7 +1602,6 @@ config_init_home(PyConfig *config) L"PYTHONHOME", "PYTHONHOME"); } - static PyStatus config_init_hash_seed(PyConfig *config) { @@ -1268,7 +1615,7 @@ config_init_hash_seed(PyConfig *config) errno = 0; seed = strtoul(seed_text, (char **)&endptr, 10); if (*endptr != '\0' - || seed > 4294967295UL + || seed > MAX_HASH_SEED || (errno == ERANGE && seed == ULONG_MAX)) { return _PyStatus_ERR("PYTHONHASHSEED must be \"random\" " @@ -2532,58 +2879,7 @@ PyConfig_Read(PyConfig *config) goto done; } - /* Check config consistency */ - assert(config->isolated >= 0); - assert(config->use_environment >= 0); - assert(config->dev_mode >= 0); - assert(config->install_signal_handlers >= 0); - assert(config->use_hash_seed >= 0); - assert(config->faulthandler >= 0); - assert(config->tracemalloc >= 0); - assert(config->site_import >= 0); - assert(config->bytes_warning >= 0); - assert(config->inspect >= 0); - assert(config->interactive >= 0); - assert(config->optimization_level >= 0); - assert(config->parser_debug >= 0); - assert(config->write_bytecode >= 0); - assert(config->verbose >= 0); - assert(config->quiet >= 0); - assert(config->user_site_directory >= 0); - assert(config->parse_argv >= 0); - assert(config->configure_c_stdio >= 0); - assert(config->buffered_stdio >= 0); - assert(config->program_name != NULL); - assert(_PyWideStringList_CheckConsistency(&config->argv)); - /* sys.argv must be non-empty: empty argv is replaced with [''] */ - assert(config->argv.length >= 1); - assert(_PyWideStringList_CheckConsistency(&config->xoptions)); - assert(_PyWideStringList_CheckConsistency(&config->warnoptions)); - assert(_PyWideStringList_CheckConsistency(&config->module_search_paths)); - if (config->_install_importlib) { - assert(config->module_search_paths_set != 0); - /* don't check config->module_search_paths */ - assert(config->executable != NULL); - assert(config->base_executable != NULL); - assert(config->prefix != NULL); - assert(config->base_prefix != NULL); - assert(config->exec_prefix != NULL); - assert(config->base_exec_prefix != NULL); - } - assert(config->platlibdir != NULL); - assert(config->filesystem_encoding != NULL); - assert(config->filesystem_errors != NULL); - assert(config->stdio_encoding != NULL); - assert(config->stdio_errors != NULL); -#ifdef MS_WINDOWS - assert(config->legacy_windows_stdio >= 0); -#endif - /* -c and -m options are exclusive */ - assert(!(config->run_command != NULL && config->run_module != NULL)); - assert(config->check_hash_pycs_mode != NULL); - assert(config->_install_importlib >= 0); - assert(config->pathconfig_warnings >= 0); - assert(_PyWideStringList_CheckConsistency(&config->orig_argv)); + assert(config_check_consistency(config)); status = _PyStatus_OK(); @@ -2628,7 +2924,7 @@ _Py_GetConfigsAsDict(void) /* core config */ const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); - dict = config_as_dict(config); + dict = _PyConfig_AsDict(config); if (dict == NULL) { goto error; } diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 60b24946512..ae4f0eeb2ee 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2922,7 +2922,9 @@ _PySys_UpdateConfig(PyThreadState *tstate) #define SET_SYS_FROM_WSTR(KEY, VALUE) \ SET_SYS(KEY, PyUnicode_FromWideChar(VALUE, -1)); - COPY_LIST("path", config->module_search_paths); + if (config->module_search_paths_set) { + COPY_LIST("path", config->module_search_paths); + } SET_SYS_FROM_WSTR("executable", config->executable); SET_SYS_FROM_WSTR("_base_executable", config->base_executable); From dc42af8fd16b10127ce1fc93c13bc1bfd2674aa2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 5 Nov 2020 18:58:07 +0100 Subject: [PATCH 36/61] bpo-42260: PyConfig_Read() only parses argv once (GH-23168) The PyConfig_Read() function now only parses PyConfig.argv arguments once: PyConfig.parse_argv is set to 2 after arguments are parsed. Since Python arguments are strippped from PyConfig.argv, parsing arguments twice would parse the application options as Python options. * Rework the PyConfig documentation. * Fix _testinternalcapi.set_config() error handling. * SetConfigTests no longer needs parse_argv=0 when restoring the old configuration. --- Doc/c-api/init_config.rst | 190 ++++++++++-------- Lib/test/_test_embed_set_config.py | 8 +- Lib/test/test_embed.py | 10 +- .../2020-11-05-18-02-07.bpo-42260.pAeaNR.rst | 5 + Modules/_testinternalcapi.c | 9 +- Python/initconfig.c | 11 +- 6 files changed, 131 insertions(+), 102 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst index c957d6c0f72..edfeba5db7d 100644 --- a/Doc/c-api/init_config.rst +++ b/Doc/c-api/init_config.rst @@ -8,55 +8,68 @@ Python Initialization Configuration .. versionadded:: 3.8 -Structures: +Python can be initialized with :c:func:`Py_InitializeFromConfig` and the +:c:type:`PyConfig` structure. It can be preinitialized with +:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure. -* :c:type:`PyConfig` -* :c:type:`PyPreConfig` -* :c:type:`PyStatus` -* :c:type:`PyWideStringList` +There are two kinds of configuration: -Functions: +* The :ref:`Python Configuration ` can be used to build a + customized Python which behaves as the regular Python. For example, + environments variables and command line arguments are used to configure + Python. -* :c:func:`PyConfig_Clear` -* :c:func:`PyConfig_InitIsolatedConfig` -* :c:func:`PyConfig_InitPythonConfig` -* :c:func:`PyConfig_Read` -* :c:func:`PyConfig_SetArgv` -* :c:func:`PyConfig_SetBytesArgv` -* :c:func:`PyConfig_SetBytesString` -* :c:func:`PyConfig_SetString` -* :c:func:`PyConfig_SetWideStringList` -* :c:func:`PyPreConfig_InitIsolatedConfig` -* :c:func:`PyPreConfig_InitPythonConfig` -* :c:func:`PyStatus_Error` -* :c:func:`PyStatus_Exception` -* :c:func:`PyStatus_Exit` -* :c:func:`PyStatus_IsError` -* :c:func:`PyStatus_IsExit` -* :c:func:`PyStatus_NoMemory` -* :c:func:`PyStatus_Ok` -* :c:func:`PyWideStringList_Append` -* :c:func:`PyWideStringList_Insert` -* :c:func:`Py_ExitStatusException` -* :c:func:`Py_InitializeFromConfig` -* :c:func:`Py_PreInitialize` -* :c:func:`Py_PreInitializeFromArgs` -* :c:func:`Py_PreInitializeFromBytesArgs` -* :c:func:`Py_RunMain` -* :c:func:`Py_GetArgcArgv` - -The preconfiguration (``PyPreConfig`` type) is stored in -``_PyRuntime.preconfig`` and the configuration (``PyConfig`` type) is stored in -``PyInterpreterState.config``. +* The :ref:`Isolated Configuration ` can be used to embed + Python into an application. It isolates Python from the system. For example, + environments variables are ignored, the LC_CTYPE locale is left unchanged and + no signal handler is registred. See also :ref:`Initialization, Finalization, and Threads `. .. seealso:: :pep:`587` "Python Initialization Configuration". +Example +======= + +Example of customized Python always running in isolated mode:: + + int main(int argc, char **argv) + { + PyStatus status; + + PyConfig config; + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + + /* Decode command line arguments. + Implicitly preinitialize Python (in isolated mode). */ + status = PyConfig_SetBytesArgv(&config, argc, argv); + if (PyStatus_Exception(status)) { + goto exception; + } + + status = Py_InitializeFromConfig(&config); + if (PyStatus_Exception(status)) { + goto exception; + } + PyConfig_Clear(&config); + + return Py_RunMain(); + + exception: + PyConfig_Clear(&config); + if (PyStatus_IsExit(status)) { + return status.exitcode; + } + /* Display the error message and exit the process with + non-zero exit code */ + Py_ExitStatusException(status); + } + PyWideStringList ----------------- +================ .. c:type:: PyWideStringList @@ -95,7 +108,7 @@ PyWideStringList List items. PyStatus --------- +======== .. c:type:: PyStatus @@ -187,7 +200,7 @@ Example:: PyPreConfig ------------ +=========== .. c:type:: PyPreConfig @@ -317,7 +330,7 @@ PyPreConfig .. _c-preinit: Preinitialize Python with PyPreConfig -------------------------------------- +===================================== The preinitialization of Python: @@ -326,12 +339,17 @@ The preinitialization of Python: * Set the :ref:`Python UTF-8 Mode ` (:c:member:`PyPreConfig.utf8_mode`) +The current preconfiguration (``PyPreConfig`` type) is stored in +``_PyRuntime.preconfig``. + Functions to preinitialize Python: .. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) Preinitialize Python from *preconfig* preconfiguration. + *preconfig* must not be ``NULL``. + .. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) Preinitialize Python from *preconfig* preconfiguration. @@ -339,6 +357,8 @@ Functions to preinitialize Python: Parse *argv* command line arguments (bytes strings) if :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + *preconfig* must not be ``NULL``. + .. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) Preinitialize Python from *preconfig* preconfiguration. @@ -346,6 +366,8 @@ Functions to preinitialize Python: Parse *argv* command line arguments (wide strings) if :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. + *preconfig* must not be ``NULL``. + The caller is responsible to handle exceptions (error or exit) using :c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. @@ -388,7 +410,7 @@ the :ref:`Python UTF-8 Mode `:: PyConfig --------- +======== .. c:type:: PyConfig @@ -449,8 +471,20 @@ PyConfig Fields which are already initialized are left unchanged. + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. + :ref:`Preinitialize Python ` if needed. + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed once, + :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are + parsed, and arguments are only parsed if + :c:member:`PyConfig.parse_argv` equals ``1``. + .. c:function:: void PyConfig_Clear(PyConfig *config) Release configuration memory. @@ -833,7 +867,7 @@ PyConfig If :c:member:`~PyConfig.orig_argv` list is empty and :c:member:`~PyConfig.argv` is not a list only containing an empty - string, :c:func:`PyConfig_Read()` copies :c:member:`~PyConfig.argv` into + string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into :c:member:`~PyConfig.orig_argv` before modifying :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is non-zero). @@ -849,12 +883,22 @@ PyConfig Parse command line arguments? - If non-zero, parse :c:member:`~PyConfig.argv` the same way the regular + If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular Python parses :ref:`command line arguments `, and strip Python arguments from :c:member:`~PyConfig.argv`. + The :c:func:`PyConfig_Read` function only parses + :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` + is set to ``2`` after arguments are parsed. Since Python arguments are + strippped from :c:member:`PyConfig.argv`, parsing arguments twice would + parse the application options as Python options. + Default: ``1`` in Python mode, ``0`` in isolated mode. + .. versionchanged:: 3.10 + The :c:member:`PyConfig.argv` arguments are now only parsed if + :c:member:`PyConfig.parse_argv` equals to ``1``. + .. c:member:: int parser_debug Parser debug mode. If greater than 0, turn on parser debugging output (for expert only, depending @@ -1108,7 +1152,7 @@ the :option:`-X` command line option. Initialization with PyConfig ----------------------------- +============================ Function to initialize Python: @@ -1123,6 +1167,9 @@ If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or :c:func:`PyImport_ExtendInittab` are used, they must be set or called after Python preinitialization and before the Python initialization. +The current configuration (``PyConfig`` type) is stored in +``PyInterpreterState.config``. + Example setting the program name:: void init_python(void) @@ -1136,17 +1183,17 @@ Example setting the program name:: status = PyConfig_SetString(&config, &config.program_name, L"/path/to/my_program"); if (PyStatus_Exception(status)) { - goto fail; + goto exception; } status = Py_InitializeFromConfig(&config); if (PyStatus_Exception(status)) { - goto fail; + goto exception; } PyConfig_Clear(&config); return; - fail: + exception: PyConfig_Clear(&config); Py_ExitStatusException(status); } @@ -1202,7 +1249,7 @@ configuration, and then override some parameters:: .. _init-isolated-conf: Isolated Configuration ----------------------- +====================== :c:func:`PyPreConfig_InitIsolatedConfig` and :c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to @@ -1223,7 +1270,7 @@ configuration. .. _init-python-config: Python Configuration --------------------- +==================== :c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` functions create a configuration to build a customized Python which behaves as @@ -1237,46 +1284,11 @@ and :ref:`Python UTF-8 Mode ` (:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and :envvar:`PYTHONCOERCECLOCALE` environment variables. -Example of customized Python always running in isolated mode:: - - int main(int argc, char **argv) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config.isolated = 1; - - /* Decode command line arguments. - Implicitly preinitialize Python (in isolated mode). */ - status = PyConfig_SetBytesArgv(&config, argc, argv); - if (PyStatus_Exception(status)) { - goto fail; - } - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto fail; - } - PyConfig_Clear(&config); - - return Py_RunMain(); - - fail: - PyConfig_Clear(&config); - if (PyStatus_IsExit(status)) { - return status.exitcode; - } - /* Display the error message and exit the process with - non-zero exit code */ - Py_ExitStatusException(status); - } - .. _init-path-config: Path Configuration ------------------- +================== :c:type:`PyConfig` contains multiple fields for the path configuration: @@ -1356,7 +1368,7 @@ The ``__PYVENV_LAUNCHER__`` environment variable is used to set Py_RunMain() ------------- +============ .. c:function:: int Py_RunMain(void) @@ -1376,7 +1388,7 @@ customized Python always running in isolated mode using Py_GetArgcArgv() ----------------- +================ .. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) @@ -1386,7 +1398,7 @@ Py_GetArgcArgv() Multi-Phase Initialization Private Provisional API --------------------------------------------------- +================================================== This section is a private provisional API introducing multi-phase initialization, the core feature of the :pep:`432`: diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index 7c913811ded..a19f8db1584 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -20,7 +20,7 @@ class SetConfigTests(unittest.TestCase): self.sys_copy = dict(sys.__dict__) def tearDown(self): - self.set_config(parse_argv=0) + _testinternalcapi.set_config(self.old_config) sys.__dict__.clear() sys.__dict__.update(self.sys_copy) @@ -234,6 +234,12 @@ class SetConfigTests(unittest.TestCase): self.assertEqual(sys.argv, ['python_program', 'args']) self.assertEqual(sys.orig_argv, ['orig', 'orig_args']) + self.set_config(parse_argv=0, + argv=[], + orig_argv=[]) + self.assertEqual(sys.argv, ['']) + self.assertEqual(sys.orig_argv, []) + def test_pycache_prefix(self): self.check(pycache_prefix=None) self.check(pycache_prefix="pycache_prefix") diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 91820615193..a7d912178a2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -422,7 +422,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): CONFIG_PYTHON = dict(CONFIG_COMPAT, _config_init=API_PYTHON, configure_c_stdio=1, - parse_argv=1, + parse_argv=2, ) CONFIG_ISOLATED = dict(CONFIG_COMPAT, _config_init=API_ISOLATED, @@ -800,7 +800,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): '-X', 'cmdline_xoption', '-c', 'pass', 'arg2'], - 'parse_argv': 1, + 'parse_argv': 2, 'xoptions': [ 'config_xoption1=3', 'config_xoption2=', @@ -1045,7 +1045,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'orig_argv': ['python3', '-c', code, 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 1, + 'parse_argv': 2, } self.check_all_configs("test_init_run_main", config, api=API_PYTHON) @@ -1059,7 +1059,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): 'arg2'], 'program_name': './python3', 'run_command': code + '\n', - 'parse_argv': 1, + 'parse_argv': 2, '_init_main': 0, } self.check_all_configs("test_init_main", config, @@ -1068,7 +1068,7 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase): def test_init_parse_argv(self): config = { - 'parse_argv': 1, + 'parse_argv': 2, 'argv': ['-c', 'arg1', '-v', 'arg3'], 'orig_argv': ['./argv0', '-E', '-c', 'pass', 'arg1', '-v', 'arg3'], 'program_name': './argv0', diff --git a/Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst b/Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst new file mode 100644 index 00000000000..0d6a277db88 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-05-18-02-07.bpo-42260.pAeaNR.rst @@ -0,0 +1,5 @@ +The :c:func:`PyConfig_Read` function now only parses :c:member:`PyConfig.argv` +arguments once: :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments +are parsed. Since Python arguments are strippped from +:c:member:`PyConfig.argv`, parsing arguments twice would parse the application +options as Python options. diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index be144bfba02..df4725ea0a1 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -253,14 +253,17 @@ test_set_config(PyObject *Py_UNUSED(self), PyObject *dict) PyConfig config; PyConfig_InitIsolatedConfig(&config); if (_PyConfig_FromDict(&config, dict) < 0) { - PyConfig_Clear(&config); - return NULL; + goto error; } if (_PyInterpreterState_SetConfig(&config) < 0) { - return NULL; + goto error; } PyConfig_Clear(&config); Py_RETURN_NONE; + +error: + PyConfig_Clear(&config); + return NULL; } diff --git a/Python/initconfig.c b/Python/initconfig.c index d54d5b7a999..e0811b56cb3 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -1325,8 +1325,6 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_UINT(_init_main); GET_UINT(_isolated_interpreter); - assert(config_check_consistency(config)); - #undef CHECK_VALUE #undef GET_UINT #undef GET_WSTR @@ -2145,6 +2143,11 @@ config_read(PyConfig *config) config->configure_c_stdio = 1; } + // Only parse arguments once. + if (config->parse_argv == 1) { + config->parse_argv = 2; + } + return _PyStatus_OK(); } @@ -2635,7 +2638,7 @@ core_read_precmdline(PyConfig *config, _PyPreCmdline *precmdline) { PyStatus status; - if (config->parse_argv) { + if (config->parse_argv == 1) { if (_PyWideStringList_Copy(&precmdline->argv, &config->argv) < 0) { return _PyStatus_NO_MEMORY(); } @@ -2713,7 +2716,7 @@ config_read_cmdline(PyConfig *config) } } - if (config->parse_argv) { + if (config->parse_argv == 1) { Py_ssize_t opt_index; status = config_parse_cmdline(config, &cmdline_warnoptions, &opt_index); if (_PyStatus_EXCEPTION(status)) { From 91e93794d5dd1aa91fbe142099c2955e0c4c1660 Mon Sep 17 00:00:00 2001 From: Zackery Spytz Date: Thu, 5 Nov 2020 15:18:44 -0700 Subject: [PATCH 37/61] bpo-26389: Allow passing an exception object in the traceback module (GH-22610) The format_exception(), format_exception_only(), and print_exception() functions can now take an exception object as a positional-only argument. Co-Authored-By: Matthias Bussonnier --- Doc/library/traceback.rst | 43 ++++++++++++++----- Doc/whatsnew/3.10.rst | 18 ++++++++ Lib/test/test_traceback.py | 20 +++++++++ Lib/traceback.py | 36 ++++++++++------ .../2020-10-08-23-51-55.bpo-26389.uga44e.rst | 4 ++ 5 files changed, 97 insertions(+), 24 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-08-23-51-55.bpo-26389.uga44e.rst diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 462a6a5566e..c233f18d30a 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -36,7 +36,8 @@ The module defines the following functions: Added negative *limit* support. -.. function:: print_exception(etype, value, tb, limit=None, file=None, chain=True) +.. function:: print_exception(exc, /[, value, tb], limit=None, \ + file=None, chain=True) Print exception information and stack trace entries from traceback object *tb* to *file*. This differs from :func:`print_tb` in the following @@ -45,7 +46,7 @@ The module defines the following functions: * if *tb* is not ``None``, it prints a header ``Traceback (most recent call last):`` - * it prints the exception *etype* and *value* after the stack trace + * it prints the exception type and *value* after the stack trace .. index:: single: ^ (caret); marker @@ -53,6 +54,10 @@ The module defines the following functions: format, it prints the line where the syntax error occurred with a caret indicating the approximate position of the error. + Since Python 3.10, instead of passing *value* and *tb*, an exception object + can be passed as the first argument. If *value* and *tb* are provided, the + first argument is ignored in order to provide backwards compatibility. + The optional *limit* argument has the same meaning as for :func:`print_tb`. If *chain* is true (the default), then chained exceptions (the :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be @@ -62,6 +67,10 @@ The module defines the following functions: .. versionchanged:: 3.5 The *etype* argument is ignored and inferred from the type of *value*. + .. versionchanged:: 3.10 + The *etype* parameter has been renamed to *exc* and is now + positional-only. + .. function:: print_exc(limit=None, file=None, chain=True) @@ -121,18 +130,26 @@ The module defines the following functions: text line is not ``None``. -.. function:: format_exception_only(etype, value) +.. function:: format_exception_only(exc, /[, value]) - Format the exception part of a traceback. The arguments are the exception - type and value such as given by ``sys.last_type`` and ``sys.last_value``. - The return value is a list of strings, each ending in a newline. Normally, - the list contains a single string; however, for :exc:`SyntaxError` - exceptions, it contains several lines that (when printed) display detailed - information about where the syntax error occurred. The message indicating - which exception occurred is the always last string in the list. + Format the exception part of a traceback using an exception value such as + given by ``sys.last_value``. The return value is a list of strings, each + ending in a newline. Normally, the list contains a single string; however, + for :exc:`SyntaxError` exceptions, it contains several lines that (when + printed) display detailed information about where the syntax error occurred. + The message indicating which exception occurred is the always last string in + the list. + + Since Python 3.10, instead of passing *value*, an exception object + can be passed as the first argument. If *value* is provided, the first + argument is ignored in order to provide backwards compatibility. + + .. versionchanged:: 3.10 + The *etype* parameter has been renamed to *exc* and is now + positional-only. -.. function:: format_exception(etype, value, tb, limit=None, chain=True) +.. function:: format_exception(exc, /[, value, tb], limit=None, chain=True) Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments to :func:`print_exception`. The @@ -143,6 +160,10 @@ The module defines the following functions: .. versionchanged:: 3.5 The *etype* argument is ignored and inferred from the type of *value*. + .. versionchanged:: 3.10 + This function's behavior and signature were modified to match + :func:`print_exception`. + .. function:: format_exc(limit=None, chain=True) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index bac1a2e6783..0ed7084ccd2 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -232,6 +232,15 @@ retrieve the functions set by :func:`threading.settrace` and :func:`threading.setprofile` respectively. (Contributed by Mario Corchero in :issue:`42251`.) +traceback +--------- + +The :func:`~traceback.format_exception`, +:func:`~traceback.format_exception_only`, and +:func:`~traceback.print_exception` functions can now take an exception object +as a positional-only argument. +(Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + types ----- @@ -328,6 +337,15 @@ This section lists previously described changes and other bugfixes that may require changes to your code. +Changes in the Python API +------------------------- + +* The *etype* parameters of the :func:`~traceback.format_exception`, + :func:`~traceback.format_exception_only`, and + :func:`~traceback.print_exception` functions in the :mod:`traceback` module + have been renamed to *exc*. + (Contributed by Zackery Spytz and Matthias Bussonnier in :issue:`26389`.) + Build Changes ============= diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 730596efd8b..91688ff72bb 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -212,6 +212,26 @@ class TracebackCases(unittest.TestCase): ) self.assertEqual(output.getvalue(), "Exception: projector\n") + def test_print_exception_exc(self): + output = StringIO() + traceback.print_exception(Exception("projector"), file=output) + self.assertEqual(output.getvalue(), "Exception: projector\n") + + def test_format_exception_exc(self): + e = Exception("projector") + output = traceback.format_exception(e) + self.assertEqual(output, ["Exception: projector\n"]) + with self.assertRaisesRegex(ValueError, 'Both or neither'): + traceback.format_exception(e.__class__, e) + with self.assertRaisesRegex(ValueError, 'Both or neither'): + traceback.format_exception(e.__class__, tb=e.__traceback__) + with self.assertRaisesRegex(TypeError, 'positional-only'): + traceback.format_exception(exc=e) + + def test_format_exception_only_exc(self): + output = traceback.format_exception_only(Exception("projector")) + self.assertEqual(output, ["Exception: projector\n"]) + class TracebackFormatTests(unittest.TestCase): diff --git a/Lib/traceback.py b/Lib/traceback.py index a19e38718b1..d2d93c8a32a 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -84,7 +84,19 @@ _context_message = ( "another exception occurred:\n\n") -def print_exception(etype, value, tb, limit=None, file=None, chain=True): +_sentinel = object() + + +def _parse_value_tb(exc, value, tb): + if (value is _sentinel) != (tb is _sentinel): + raise ValueError("Both or neither of value and tb must be given") + if value is tb is _sentinel: + return exc, exc.__traceback__ + return value, tb + + +def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ + file=None, chain=True): """Print exception up to 'limit' stack trace entries from 'tb' to 'file'. This differs from print_tb() in the following ways: (1) if @@ -95,9 +107,7 @@ def print_exception(etype, value, tb, limit=None, file=None, chain=True): occurred with a caret on the next line indicating the approximate position of the error. """ - # format_exception has ignored etype for some time, and code such as cgitb - # passes in bogus values as a result. For compatibility with such code we - # ignore it here (rather than in the new TracebackException API). + value, tb = _parse_value_tb(exc, value, tb) if file is None: file = sys.stderr for line in TracebackException( @@ -105,7 +115,8 @@ def print_exception(etype, value, tb, limit=None, file=None, chain=True): print(line, file=file, end="") -def format_exception(etype, value, tb, limit=None, chain=True): +def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \ + chain=True): """Format a stack trace and the exception information. The arguments have the same meaning as the corresponding arguments @@ -114,19 +125,15 @@ def format_exception(etype, value, tb, limit=None, chain=True): these lines are concatenated and printed, exactly the same text is printed as does print_exception(). """ - # format_exception has ignored etype for some time, and code such as cgitb - # passes in bogus values as a result. For compatibility with such code we - # ignore it here (rather than in the new TracebackException API). + value, tb = _parse_value_tb(exc, value, tb) return list(TracebackException( type(value), value, tb, limit=limit).format(chain=chain)) -def format_exception_only(etype, value): +def format_exception_only(exc, /, value=_sentinel): """Format the exception part of a traceback. - The arguments are the exception type and value such as given by - sys.last_type and sys.last_value. The return value is a list of - strings, each ending in a newline. + The return value is a list of strings, each ending in a newline. Normally, the list contains a single string; however, for SyntaxError exceptions, it contains several lines that (when @@ -137,7 +144,10 @@ def format_exception_only(etype, value): string in the list. """ - return list(TracebackException(etype, value, None).format_exception_only()) + if value is _sentinel: + value = exc + return list(TracebackException( + type(value), value, None).format_exception_only()) # -- not official API but folk probably use these two functions. diff --git a/Misc/NEWS.d/next/Library/2020-10-08-23-51-55.bpo-26389.uga44e.rst b/Misc/NEWS.d/next/Library/2020-10-08-23-51-55.bpo-26389.uga44e.rst new file mode 100644 index 00000000000..a721a0d7cd0 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-08-23-51-55.bpo-26389.uga44e.rst @@ -0,0 +1,4 @@ +The :func:`traceback.format_exception`, +:func:`traceback.format_exception_only`, and +:func:`traceback.print_exception` functions can now take an exception object +as a positional-only argument. From bde33e428d5b5f88ec7667598fd27d1091840537 Mon Sep 17 00:00:00 2001 From: Inada Naoki Date: Fri, 6 Nov 2020 11:45:01 +0900 Subject: [PATCH 38/61] bpo-42179: Doc/tutorial: Remove mention of __cause__ (GH-23162) --- Doc/tutorial/errors.rst | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst index 0ce96466e8c..efe44da3043 100644 --- a/Doc/tutorial/errors.rst +++ b/Doc/tutorial/errors.rst @@ -273,15 +273,15 @@ Exception Chaining ================== The :keyword:`raise` statement allows an optional :keyword:`from` which enables -chaining exceptions by setting the ``__cause__`` attribute of the raised -exception. For example:: +chaining exceptions. For example:: - raise RuntimeError from OSError + # exc must be exception instance or None. + raise RuntimeError from exc This can be useful when you are transforming exceptions. For example:: >>> def func(): - ... raise IOError + ... raise IOError ... >>> try: ... func() @@ -297,12 +297,11 @@ This can be useful when you are transforming exceptions. For example:: Traceback (most recent call last): File "", line 4, in - RuntimeError + RuntimeError: Failed to open database -The expression following the :keyword:`from` must be either an exception or -``None``. Exception chaining happens automatically when an exception is raised -inside an exception handler or :keyword:`finally` section. Exception chaining -can be disabled by using ``from None`` idiom: +Exception chaining happens automatically when an exception is raised inside an +:keyword:`except` or :keyword:`finally` section. Exception chaining can be +disabled by using ``from None`` idiom: >>> try: ... open('database.sqlite') @@ -313,6 +312,8 @@ can be disabled by using ``from None`` idiom: File "", line 4, in RuntimeError +For more information about chaining mechanics, see :ref:`bltin-exceptions`. + .. _tut-userexceptions: From 803187796f06bdc47ae74ce3d28c443e8cc2a27f Mon Sep 17 00:00:00 2001 From: Raymond Hettinger Date: Fri, 6 Nov 2020 01:30:17 -0800 Subject: [PATCH 39/61] Minor grammar edits for the descriptor howto guide (GH-#23175) --- Doc/howto/descriptor.rst | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst index fedf8a8c09e..8c2e8d56215 100644 --- a/Doc/howto/descriptor.rst +++ b/Doc/howto/descriptor.rst @@ -112,7 +112,7 @@ different, updated answers each time:: Besides showing how descriptors can run computations, this example also reveals the purpose of the parameters to :meth:`__get__`. The *self* parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is -either *g* or *s*, an instance of *Directory*. It is *obj* parameter that +either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that lets the :meth:`__get__` method learn the target directory. The *objtype* parameter is the class *Directory*. @@ -183,7 +183,7 @@ logged, but that the regular attribute *name* is not logged:: INFO:root:Accessing 'age' giving 40 40 -One major issue with this example is the private name *_age* is hardwired in +One major issue with this example is that the private name *_age* is hardwired in the *LoggedAgeAccess* class. That means that each instance can only have one logged attribute and that its name is unchangeable. In the next example, we'll fix that problem. @@ -192,7 +192,7 @@ we'll fix that problem. Customized names ---------------- -When a class uses descriptors, it can inform each descriptor about what +When a class uses descriptors, it can inform each descriptor about which variable name was used. In this example, the :class:`Person` class has two descriptor instances, @@ -233,7 +233,7 @@ be recorded, giving each descriptor its own *public_name* and *private_name*:: An interactive session shows that the :class:`Person` class has called :meth:`__set_name__` so that the field names would be recorded. Here -we call :func:`vars` to lookup the descriptor without triggering it:: +we call :func:`vars` to look up the descriptor without triggering it:: >>> vars(vars(Person)['name']) {'public_name': 'name', 'private_name': '_name'} @@ -614,8 +614,8 @@ Sometimes it is desirable for a descriptor to know what class variable name it was assigned to. When a new class is created, the :class:`type` metaclass scans the dictionary of the new class. If any of the entries are descriptors and if they define :meth:`__set_name__`, that method is called with two -arguments. The *owner* is the class where the descriptor is used, the *name* -is class variable the descriptor was assigned to. +arguments. The *owner* is the class where the descriptor is used, and the +*name* is the class variable the descriptor was assigned to. The implementation details are in :c:func:`type_new()` and :c:func:`set_names()` in :source:`Objects/typeobject.c`. @@ -703,7 +703,7 @@ Properties ---------- Calling :func:`property` is a succinct way of building a data descriptor that -triggers function calls upon access to an attribute. Its signature is:: +triggers a function call upon access to an attribute. Its signature is:: property(fget=None, fset=None, fdel=None, doc=None) -> property @@ -803,7 +803,7 @@ roughly equivalent to:: To support automatic creation of methods, functions include the :meth:`__get__` method for binding methods during attribute access. This -means that functions are non-data descriptors which return bound methods +means that functions are non-data descriptors that return bound methods during dotted lookup from an instance. Here's how it works:: class Function: @@ -1016,7 +1016,7 @@ attributes stored in ``__slots__``:: class Immutable: - __slots__ = ('_dept', '_name') # Replace instance dictionary + __slots__ = ('_dept', '_name') # Replace the instance dictionary def __init__(self, dept, name): self._dept = dept # Store to private attribute @@ -1086,7 +1086,7 @@ by member descriptors:: The :meth:`type.__new__` method takes care of adding member objects to class variables. The :meth:`object.__new__` method takes care of creating instances -that have slots instead of a instance dictionary. Here is a rough equivalent +that have slots instead of an instance dictionary. Here is a rough equivalent in pure Python:: class Type(type): From 88c2cfd9ffbcfc43fd1364f2984852a819547d43 Mon Sep 17 00:00:00 2001 From: Hai Shi Date: Sat, 7 Nov 2020 00:04:47 +0800 Subject: [PATCH 40/61] bpo-41832: PyType_FromModuleAndSpec() now accepts NULL tp_doc (GH-23123) --- Doc/c-api/type.rst | 6 +++-- Doc/whatsnew/3.10.rst | 4 +++ Lib/test/test_capi.py | 3 +++ .../2020-11-03-19-47-06.bpo-41832.dL1VJJ.rst | 2 ++ Modules/_lsprof.c | 4 +-- Modules/_testcapimodule.c | 25 +++++++++++++++++++ Objects/typeobject.c | 4 +++ 7 files changed, 44 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-11-03-19-47-06.bpo-41832.dL1VJJ.rst diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst index 73f26875d81..fcd92e38e24 100644 --- a/Doc/c-api/type.rst +++ b/Doc/c-api/type.rst @@ -169,6 +169,10 @@ The following functions and structs are used to create .. versionadded:: 3.9 + .. versionchanged:: 3.10 + + The function now accepts NULL ``tp_doc`` slot. + .. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``. @@ -259,5 +263,3 @@ The following functions and structs are used to create The desired value of the slot. In most cases, this is a pointer to a function. - - May not be ``NULL``. diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 0ed7084ccd2..1e6c7c4067f 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -401,6 +401,10 @@ New Features reference count of an object and return the object. (Contributed by Victor Stinner in :issue:`42262`.) +* The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` + slot. + (Contributed by Hai Shi in :issue:`41832`.) + Porting to Python 3.10 ---------------------- diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index db62b47100a..a4ebe4a0a1b 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -405,6 +405,9 @@ class CAPITest(unittest.TestCase): self.assertEqual(_testcapi.HeapDocCType.__doc__, "somedoc") self.assertEqual(_testcapi.HeapDocCType.__text_signature__, "(arg1, arg2)") + def test_null_type_doc(self): + self.assertEqual(_testcapi.NullTpDocType.__doc__, None) + def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self): class HeapGcCTypeSubclass(_testcapi.HeapGcCType): def __init__(self): diff --git a/Misc/NEWS.d/next/C API/2020-11-03-19-47-06.bpo-41832.dL1VJJ.rst b/Misc/NEWS.d/next/C API/2020-11-03-19-47-06.bpo-41832.dL1VJJ.rst new file mode 100644 index 00000000000..e0bce54eb93 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-11-03-19-47-06.bpo-41832.dL1VJJ.rst @@ -0,0 +1,2 @@ +The :c:func:`PyType_FromModuleAndSpec` function now accepts NULL ``tp_doc`` +slot. diff --git a/Modules/_lsprof.c b/Modules/_lsprof.c index 78d464d1481..c32699cb8ad 100644 --- a/Modules/_lsprof.c +++ b/Modules/_lsprof.c @@ -489,15 +489,15 @@ static PyStructSequence_Field profiler_subentry_fields[] = { static PyStructSequence_Desc profiler_entry_desc = { .name = "_lsprof.profiler_entry", - .doc = "", .fields = profiler_entry_fields, + .doc = NULL, .n_in_sequence = 6 }; static PyStructSequence_Desc profiler_subentry_desc = { .name = "_lsprof.profiler_subentry", - .doc = "", .fields = profiler_subentry_fields, + .doc = NULL, .n_in_sequence = 5 }; diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 28d2c124d51..22d20d220d4 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -6508,6 +6508,23 @@ static PyType_Spec HeapDocCType_spec = { HeapDocCType_slots }; +typedef struct { + PyObject_HEAD +} NullTpDocTypeObject; + +static PyType_Slot NullTpDocType_slots[] = { + {Py_tp_doc, NULL}, + {0, 0}, +}; + +static PyType_Spec NullTpDocType_spec = { + "_testcapi.NullTpDocType", + sizeof(NullTpDocTypeObject), + 0, + Py_TPFLAGS_DEFAULT, + NullTpDocType_slots +}; + PyDoc_STRVAR(heapgctype__doc__, "A heap type with GC, and with overridden dealloc.\n\n" @@ -7183,6 +7200,14 @@ PyInit__testcapi(void) } PyModule_AddObject(m, "HeapDocCType", HeapDocCType); + /* bpo-41832: Add a new type to test PyType_FromSpec() + now can accept a NULL tp_doc slot. */ + PyObject *NullTpDocType = PyType_FromSpec(&NullTpDocType_spec); + if (NullTpDocType == NULL) { + return NULL; + } + PyModule_AddObject(m, "NullTpDocType", NullTpDocType); + PyObject *HeapGcCType = PyType_FromSpec(&HeapGcCType_spec); if (HeapGcCType == NULL) { return NULL; diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 2daf374f170..3822b8cf813 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -3012,6 +3012,10 @@ PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) else if (slot->slot == Py_tp_doc) { /* For the docstring slot, which usually points to a static string literal, we need to make a copy */ + if (slot->pfunc == NULL) { + type->tp_doc = NULL; + continue; + } size_t len = strlen(slot->pfunc)+1; char *tp_doc = PyObject_MALLOC(len); if (tp_doc == NULL) { From 7c01f1540f958d4f52188b28afca721a9a6925c3 Mon Sep 17 00:00:00 2001 From: Andre Delfino Date: Fri, 6 Nov 2020 15:58:14 -0300 Subject: [PATCH 41/61] Remove outdated reference to pywin32 from platform module (GH-22005) --- Doc/library/platform.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst index 8e8e3775aaf..b293adf48e6 100644 --- a/Doc/library/platform.rst +++ b/Doc/library/platform.rst @@ -209,13 +209,6 @@ Windows Platform which means the OS version uses debugging code, i.e. code that checks arguments, ranges, etc. - .. note:: - - This function works best with Mark Hammond's - :mod:`win32all` package installed, but also on Python 2.3 and - later (support for this was added in Python 2.6). It obviously - only runs on Win32 compatible platforms. - .. function:: win32_edition() Returns a string representing the current Windows edition. Possible From 825ac383327255d38b69a753e5e41710bb3ed010 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 6 Nov 2020 18:45:56 -0800 Subject: [PATCH 42/61] bpo-42133: update parts of the stdlib to fall back to `__spec__.loader` when `__loader__` is missing (#22929) --- Doc/whatsnew/3.10.rst | 24 +++++++++++++++++++ Lib/doctest.py | 18 ++++++++------ Lib/inspect.py | 7 ++++-- Lib/linecache.py | 11 ++++++--- Lib/site.py | 11 +++++++-- .../2020-10-23-15-47-47.bpo-42133.BzizYV.rst | 2 ++ 6 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index 1e6c7c4067f..a735bf23543 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -186,6 +186,12 @@ by :func:`curses.color_content`, :func:`curses.init_color`, support is provided by the underlying ncurses library. (Contributed by Jeffrey Kintscher and Hans Petter Jansson in :issue:`36982`.) +doctest +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + encodings --------- :func:`encodings.normalize_encoding` now ignores non-ASCII characters. @@ -198,6 +204,18 @@ Added the *root_dir* and *dir_fd* parameters in :func:`~glob.glob` and :func:`~glob.iglob` which allow to specify the root directory for searching. (Contributed by Serhiy Storchaka in :issue:`38144`.) +inspect +------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + +linecache +--------- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + os -- @@ -217,6 +235,12 @@ The :mod:`shelve` module now uses :data:`pickle.DEFAULT_PROTOCOL` by default instead of :mod:`pickle` protocol ``3`` when creating shelves. (Contributed by Zackery Spytz in :issue:`34204`.) +site +---- + +When a module does not define ``__loader__``, fall back to ``__spec__.loader``. +(Contributed by Brett Cannon in :issue:`42133`.) + sys --- diff --git a/Lib/doctest.py b/Lib/doctest.py index baa503c83f8..5bb35c9715d 100644 --- a/Lib/doctest.py +++ b/Lib/doctest.py @@ -222,13 +222,17 @@ def _load_testfile(filename, package, module_relative, encoding): if module_relative: package = _normalize_module(package, 3) filename = _module_relative_path(package, filename) - if getattr(package, '__loader__', None) is not None: - if hasattr(package.__loader__, 'get_data'): - file_contents = package.__loader__.get_data(filename) - file_contents = file_contents.decode(encoding) - # get_data() opens files as 'rb', so one must do the equivalent - # conversion as universal newlines would do. - return _newline_convert(file_contents), filename + if (loader := getattr(package, '__loader__', None)) is None: + try: + loader = package.__spec__.loader + except AttributeError: + pass + if hasattr(loader, 'get_data'): + file_contents = loader.get_data(filename) + file_contents = file_contents.decode(encoding) + # get_data() opens files as 'rb', so one must do the equivalent + # conversion as universal newlines would do. + return _newline_convert(file_contents), filename with open(filename, encoding=encoding) as f: return f.read(), filename diff --git a/Lib/inspect.py b/Lib/inspect.py index ac127cbd725..7412d0e837c 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -707,10 +707,13 @@ def getsourcefile(object): if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader - if getattr(getmodule(object, filename), '__loader__', None) is not None: + module = getmodule(object, filename) + if getattr(module, '__loader__', None) is not None: + return filename + elif getattr(getattr(module, "__spec__", None), "loader", None) is not None: return filename # or it is in the linecache - if filename in linecache.cache: + elif filename in linecache.cache: return filename def getabsfile(object, _filename=None): diff --git a/Lib/linecache.py b/Lib/linecache.py index fa5dbd09eab..513b17e9998 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -165,9 +165,14 @@ def lazycache(filename, module_globals): if not filename or (filename.startswith('<') and filename.endswith('>')): return False # Try for a __loader__, if available - if module_globals and '__loader__' in module_globals: - name = module_globals.get('__name__') - loader = module_globals['__loader__'] + if module_globals and '__name__' in module_globals: + name = module_globals['__name__'] + if (loader := module_globals.get('__loader__')) is None: + if spec := module_globals.get('__spec__'): + try: + loader = spec.loader + except AttributeError: + pass get_source = getattr(loader, 'get_source', None) if name and get_source: diff --git a/Lib/site.py b/Lib/site.py index 4d3b869fff7..3a0f619d71c 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -105,8 +105,15 @@ def makepath(*paths): def abs_paths(): """Set all module __file__ and __cached__ attributes to an absolute path""" for m in set(sys.modules.values()): - if (getattr(getattr(m, '__loader__', None), '__module__', None) not in - ('_frozen_importlib', '_frozen_importlib_external')): + loader_module = None + try: + loader_module = m.__loader__.__module__ + except AttributeError: + try: + loader_module = m.__spec__.loader.__module__ + except AttributeError: + pass + if loader_module not in {'_frozen_importlib', '_frozen_importlib_external'}: continue # don't mess with a PEP 302-supplied __file__ try: m.__file__ = os.path.abspath(m.__file__) diff --git a/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst b/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst new file mode 100644 index 00000000000..f3cfa1a8dce --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-23-15-47-47.bpo-42133.BzizYV.rst @@ -0,0 +1,2 @@ +Update various modules in the stdlib to fall back on `__spec__.loader` when +`__loader__` isn't defined on a module. From ee2549c2ba8bae00f2b2fea8a39c6dfbd1d06520 Mon Sep 17 00:00:00 2001 From: Julien Palard Date: Sat, 7 Nov 2020 12:28:31 +0100 Subject: [PATCH 43/61] bpo-41028: Doc: Move switchers to docsbuild-scripts. (GH-20969) --- Doc/Makefile | 4 +- Doc/tools/static/switchers.js | 156 ------------------ Doc/tools/templates/dummy.html | 9 + Doc/tools/templates/indexsidebar.html | 8 +- Doc/tools/templates/layout.html | 10 +- .../2020-06-18-23-37-03.bpo-41028.vM8bC8.rst | 2 + 6 files changed, 16 insertions(+), 173 deletions(-) delete mode 100644 Doc/tools/static/switchers.js create mode 100644 Misc/NEWS.d/next/Documentation/2020-06-18-23-37-03.bpo-41028.vM8bC8.rst diff --git a/Doc/Makefile b/Doc/Makefile index f653d70674e..6bf1f408b56 100644 --- a/Doc/Makefile +++ b/Doc/Makefile @@ -215,12 +215,12 @@ serve: # for development releases: always build autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' + make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' -make suspicious # for quick rebuilds (HTML only) autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1 -A switchers=1' + make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' # for stable releases: only build if not in pre-release stage (alpha, beta) # release candidate downloads are okay, since the stable tree can be in that stage diff --git a/Doc/tools/static/switchers.js b/Doc/tools/static/switchers.js deleted file mode 100644 index 1a1c7d0fa57..00000000000 --- a/Doc/tools/static/switchers.js +++ /dev/null @@ -1,156 +0,0 @@ -(function() { - 'use strict'; - - // Parses versions in URL segments like: - // "3", "dev", "release/2.7" or "3.6rc2" - var version_regexs = [ - '(?:\\d)', - '(?:\\d\\.\\d[\\w\\d\\.]*)', - '(?:dev)', - '(?:release/\\d.\\d[\\x\\d\\.]*)']; - - var all_versions = { - '3.10': 'dev (3.10)', - '3.9': 'pre (3.9)', - '3.8': '3.8', - '3.7': '3.7', - '3.6': '3.6', - '2.7': '2.7', - }; - - var all_languages = { - 'en': 'English', - 'fr': 'French', - 'ja': 'Japanese', - 'ko': 'Korean', - 'pt-br': 'Brazilian Portuguese', - 'zh-cn': 'Simplified Chinese', - }; - - function build_version_select(current_version, current_release) { - var buf = [''); - return buf.join(''); - } - - function build_language_select(current_language) { - var buf = [''); - return buf.join(''); - } - - function navigate_to_first_existing(urls) { - // Navigate to the first existing URL in urls. - var url = urls.shift(); - if (urls.length == 0) { - window.location.href = url; - return; - } - $.ajax({ - url: url, - success: function() { - window.location.href = url; - }, - error: function() { - navigate_to_first_existing(urls); - } - }); - } - - function on_version_switch() { - var selected_version = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + current_language + selected_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - url.replace('.org/' + current_language + current_version, - '.org/' + selected_version), - 'https://docs.python.org/' + current_language + selected_version, - 'https://docs.python.org/' + selected_version, - 'https://docs.python.org/' - ]); - } - } - - function on_language_switch() { - var selected_language = $(this).children('option:selected').attr('value') + '/'; - var url = window.location.href; - var current_language = language_segment_from_url(url); - var current_version = version_segment_in_url(url); - if (selected_language == 'en/') // Special 'default' case for english. - selected_language = ''; - var new_url = url.replace('.org/' + current_language + current_version, - '.org/' + selected_language + current_version); - if (new_url != url) { - navigate_to_first_existing([ - new_url, - 'https://docs.python.org/' - ]); - } - } - - // Returns the path segment of the language as a string, like 'fr/' - // or '' if not found. - function language_segment_from_url(url) { - var language_regexp = '\.org/([a-z]{2}(?:-[a-z]{2})?/)'; - var match = url.match(language_regexp); - if (match !== null) - return match[1]; - return ''; - } - - // Returns the path segment of the version as a string, like '3.6/' - // or '' if not found. - function version_segment_in_url(url) { - var language_segment = '(?:[a-z]{2}(?:-[a-z]{2})?/)'; - var version_segment = '(?:(?:' + version_regexs.join('|') + ')/)'; - var version_regexp = '\\.org/' + language_segment + '?(' + version_segment + ')'; - var match = url.match(version_regexp); - if (match !== null) - return match[1]; - return '' - } - - $(document).ready(function() { - var release = DOCUMENTATION_OPTIONS.VERSION; - var language_segment = language_segment_from_url(window.location.href); - var current_language = language_segment.replace(/\/+$/g, '') || 'en'; - var version = release.substr(0, 3); - var version_select = build_version_select(version, release); - - $('.version_switcher_placeholder').html(version_select); - $('.version_switcher_placeholder select').bind('change', on_version_switch); - - var language_select = build_language_select(current_language); - - $('.language_switcher_placeholder').html(language_select); - $('.language_switcher_placeholder select').bind('change', on_language_switch); - }); -})(); diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html index 68ae3ad148e..3438b44377f 100644 --- a/Doc/tools/templates/dummy.html +++ b/Doc/tools/templates/dummy.html @@ -6,3 +6,12 @@ In extensions/pyspecific.py: {% trans %}CPython implementation detail:{% endtrans %} {% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} {% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} + + +In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: + +{% trans %}in development{% endtrans %} +{% trans %}pre-release{% endtrans %} +{% trans %}stable{% endtrans %} +{% trans %}security-fixes{% endtrans %} +{% trans %}EOL{% endtrans %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index 1c1cb5484a4..f7bf6d8e491 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -2,12 +2,8 @@

{% trans %}Download these documents{% endtrans %}

{% trans %}Docs by version{% endtrans %}

diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html index 17592d74a4e..98ccf422480 100644 --- a/Doc/tools/templates/layout.html +++ b/Doc/tools/templates/layout.html @@ -12,22 +12,14 @@ {% block rootrellink %} {{ super() }} -
  • - {%- if switchers is defined %} - {{ language or 'en' }} - {{ release }} - {% trans %}Documentation {% endtrans %}{{ reldelim1 }} - {%- else %} +
  • {{ shorttitle }}{{ reldelim1 }} - {%- endif %}
    • {% endblock %} {% block extrahead %} {% if builder != "htmlhelp" %} - {% if switchers is defined and not embedded %} - {% endif %} {% if pagename == 'whatsnew/changelog' and not embedded %} {% endif %} {% endif %} diff --git a/Misc/NEWS.d/next/Documentation/2020-06-18-23-37-03.bpo-41028.vM8bC8.rst b/Misc/NEWS.d/next/Documentation/2020-06-18-23-37-03.bpo-41028.vM8bC8.rst new file mode 100644 index 00000000000..5fc4155b553 --- /dev/null +++ b/Misc/NEWS.d/next/Documentation/2020-06-18-23-37-03.bpo-41028.vM8bC8.rst @@ -0,0 +1,2 @@ +Language and version switchers, previously maintained in every cpython +branches, are now handled by docsbuild-script. From 8805a4dad201473599416b2c265802b8885f69b8 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sat, 7 Nov 2020 22:35:17 +1000 Subject: [PATCH 44/61] bpo-42282: Fold constants inside named expressions (GH-23190) * The AST optimiser wasn't descending into named expressions, so any constant subexpressions weren't being folded at compile time * Remove "default:" clauses inside the AST optimiser code to reduce the risk of similar bugs passing unnoticed in future compiler changes --- .../2020-11-07-21-02-05.bpo-42282.M1W4Wj.rst | 3 ++ Python/ast_opt.c | 52 +++++++++++++++---- 2 files changed, 45 insertions(+), 10 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-07-21-02-05.bpo-42282.M1W4Wj.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-07-21-02-05.bpo-42282.M1W4Wj.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-07-21-02-05.bpo-42282.M1W4Wj.rst new file mode 100644 index 00000000000..74f5c336238 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-07-21-02-05.bpo-42282.M1W4Wj.rst @@ -0,0 +1,3 @@ +Optimise constant subexpressions that appear as part of named expressions +(previously the AST optimiser did not descend into named expressions). +Patch by Nick Coghlan. diff --git a/Python/ast_opt.c b/Python/ast_opt.c index 22ca6f23aef..8c958ca7f13 100644 --- a/Python/ast_opt.c +++ b/Python/ast_opt.c @@ -7,6 +7,8 @@ static int make_const(expr_ty node, PyObject *val, PyArena *arena) { + // Even if no new value was calculated, make_const may still + // need to clear an error (e.g. for division by zero) if (val == NULL) { if (PyErr_ExceptionMatches(PyExc_KeyboardInterrupt)) { return 0; @@ -49,7 +51,7 @@ fold_unaryop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) of !=. Detecting such cases doesn't seem worthwhile. Python uses for 'is subset'/'is superset' operations on sets. They don't satisfy not folding laws. */ - int op = asdl_seq_GET(arg->v.Compare.ops, 0); + cmpop_ty op = asdl_seq_GET(arg->v.Compare.ops, 0); switch (op) { case Is: op = IsNot; @@ -63,8 +65,17 @@ fold_unaryop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) case NotIn: op = In; break; - default: - op = 0; + // The remaining comparison operators can't be safely inverted + case Eq: + case NotEq: + case Lt: + case LtE: + case Gt: + case GtE: + op = 0; // The AST enums leave "0" free as an "unused" marker + break; + // No default case, so the compiler will emit a warning if new + // comparison operators are added without being handled here } if (op) { asdl_seq_SET(arg->v.Compare.ops, 0, op); @@ -224,7 +235,7 @@ fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) PyObject *lv = lhs->v.Constant.value; PyObject *rv = rhs->v.Constant.value; - PyObject *newval; + PyObject *newval = NULL; switch (node->v.BinOp.op) { case Add: @@ -263,8 +274,11 @@ fold_binop(expr_ty node, PyArena *arena, _PyASTOptimizeState *state) case BitAnd: newval = PyNumber_And(lv, rv); break; - default: // Unknown operator + // No builtin constants implement the following operators + case MatMult: return 1; + // No default case, so the compiler will emit a warning if new binary + // operators are added without being handled here } return make_const(node, newval, arena); @@ -457,8 +471,11 @@ astfold_mod(mod_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case Expression_kind: CALL(astfold_expr, expr_ty, node_->v.Expression.body); break; - default: + // The following top level nodes don't participate in constant folding + case FunctionType_kind: break; + // No default case, so the compiler will emit a warning if new top level + // compilation nodes are added without being handled here } return 1; } @@ -567,8 +584,14 @@ astfold_expr(expr_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) return make_const(node_, PyBool_FromLong(!state->optimize), ctx_); } break; - default: + case NamedExpr_kind: + CALL(astfold_expr, expr_ty, node_->v.NamedExpr.value); break; + case Constant_kind: + // Already a constant, nothing further to do + break; + // No default case, so the compiler will emit a warning if new expression + // kinds are added without being handled here } return 1; } @@ -686,8 +709,17 @@ astfold_stmt(stmt_ty node_, PyArena *ctx_, _PyASTOptimizeState *state) case Expr_kind: CALL(astfold_expr, expr_ty, node_->v.Expr.value); break; - default: + // The following statements don't contain any subexpressions to be folded + case Import_kind: + case ImportFrom_kind: + case Global_kind: + case Nonlocal_kind: + case Pass_kind: + case Break_kind: + case Continue_kind: break; + // No default case, so the compiler will emit a warning if new statement + // kinds are added without being handled here } return 1; } @@ -700,8 +732,8 @@ astfold_excepthandler(excepthandler_ty node_, PyArena *ctx_, _PyASTOptimizeState CALL_OPT(astfold_expr, expr_ty, node_->v.ExceptHandler.type); CALL_SEQ(astfold_stmt, stmt, node_->v.ExceptHandler.body); break; - default: - break; + // No default case, so the compiler will emit a warning if new handler + // kinds are added without being handled here } return 1; } From 01c6aa43dc56b3b64d584c58a49c86f816c05a91 Mon Sep 17 00:00:00 2001 From: Erlend Egeberg Aasland Date: Sat, 7 Nov 2020 20:18:37 +0100 Subject: [PATCH 45/61] bpo-40077: Convert _queuemodule to use heap types (GH-23136) @vstinner / @corona10, would you mind reviewing this? --- .../2020-11-03-21-58-27.bpo-40077.a9qM1j.rst | 1 + Modules/_queuemodule.c | 184 +++++++++++------- Modules/clinic/_queuemodule.c.h | 58 +++--- 3 files changed, 138 insertions(+), 105 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-03-21-58-27.bpo-40077.a9qM1j.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-03-21-58-27.bpo-40077.a9qM1j.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-03-21-58-27.bpo-40077.a9qM1j.rst new file mode 100644 index 00000000000..369ba6b63ce --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-03-21-58-27.bpo-40077.a9qM1j.rst @@ -0,0 +1 @@ +Convert :mod:`queue` to use heap types. diff --git a/Modules/_queuemodule.c b/Modules/_queuemodule.c index b155ea94239..7cf73992795 100644 --- a/Modules/_queuemodule.c +++ b/Modules/_queuemodule.c @@ -1,16 +1,22 @@ #include "Python.h" +#include "structmember.h" // PyMemberDef #include // offsetof() -/*[clinic input] -module _queue -class _queue.SimpleQueue "simplequeueobject *" "&PySimpleQueueType" -[clinic start generated code]*/ -/*[clinic end generated code: output=da39a3ee5e6b4b0d input=cf49af81bcbbbea6]*/ - -static PyTypeObject PySimpleQueueType; /* forward decl */ - -static PyObject *EmptyError; +typedef struct { + PyTypeObject *SimpleQueueType; + PyObject *EmptyError; +} simplequeue_state; +static simplequeue_state * +simplequeue_get_state(PyObject *module) +{ + simplequeue_state *state = PyModule_GetState(module); + assert(state); + return state; +} +static struct PyModuleDef queuemodule; +#define simplequeue_get_state_by_type(tp) \ + (simplequeue_get_state(_PyType_GetModuleByDef(type, &queuemodule))) typedef struct { PyObject_HEAD @@ -21,10 +27,17 @@ typedef struct { PyObject *weakreflist; } simplequeueobject; +/*[clinic input] +module _queue +class _queue.SimpleQueue "simplequeueobject *" "simplequeue_get_state_by_type(type)->SimpleQueueType" +[clinic start generated code]*/ +/*[clinic end generated code: output=da39a3ee5e6b4b0d input=0a4023fe4d198c8d]*/ static void simplequeue_dealloc(simplequeueobject *self) { + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_UnTrack(self); if (self->lock != NULL) { /* Unlock the lock so it's safe to free it */ @@ -36,6 +49,7 @@ simplequeue_dealloc(simplequeueobject *self) if (self->weakreflist != NULL) PyObject_ClearWeakRefs((PyObject *) self); Py_TYPE(self)->tp_free(self); + Py_DECREF(tp); } static int @@ -155,6 +169,9 @@ simplequeue_pop_item(simplequeueobject *self) /*[clinic input] _queue.SimpleQueue.get + + cls: defining_class + / block: bool = True timeout: object = None @@ -171,9 +188,9 @@ in that case). [clinic start generated code]*/ static PyObject * -_queue_SimpleQueue_get_impl(simplequeueobject *self, int block, - PyObject *timeout) -/*[clinic end generated code: output=ec82a7157dcccd1a input=4bf691f9f01fa297]*/ +_queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, + int block, PyObject *timeout) +/*[clinic end generated code: output=1969aefa7db63666 input=5fc4d56b9a54757e]*/ { _PyTime_t endtime = 0; _PyTime_t timeout_val; @@ -225,8 +242,10 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, return NULL; } if (r == PY_LOCK_FAILURE) { + PyObject *module = PyType_GetModule(cls); + simplequeue_state *state = simplequeue_get_state(module); /* Timed out */ - PyErr_SetNone(EmptyError); + PyErr_SetNone(state->EmptyError); return NULL; } self->locked = 1; @@ -251,6 +270,9 @@ _queue_SimpleQueue_get_impl(simplequeueobject *self, int block, /*[clinic input] _queue.SimpleQueue.get_nowait + cls: defining_class + / + Remove and return an item from the queue without blocking. Only get an item if one is immediately available. Otherwise @@ -258,10 +280,11 @@ raise the Empty exception. [clinic start generated code]*/ static PyObject * -_queue_SimpleQueue_get_nowait_impl(simplequeueobject *self) -/*[clinic end generated code: output=a89731a75dbe4937 input=6fe5102db540a1b9]*/ +_queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, + PyTypeObject *cls) +/*[clinic end generated code: output=620c58e2750f8b8a input=842f732bf04216d3]*/ { - return _queue_SimpleQueue_get_impl(self, 0, Py_None); + return _queue_SimpleQueue_get_impl(self, cls, 0, Py_None); } /*[clinic input] @@ -290,6 +313,29 @@ _queue_SimpleQueue_qsize_impl(simplequeueobject *self) return PyList_GET_SIZE(self->lst) - self->lst_pos; } +static int +queue_traverse(PyObject *m, visitproc visit, void *arg) +{ + simplequeue_state *state = simplequeue_get_state(m); + Py_VISIT(state->SimpleQueueType); + Py_VISIT(state->EmptyError); + return 0; +} + +static int +queue_clear(PyObject *m) +{ + simplequeue_state *state = simplequeue_get_state(m); + Py_CLEAR(state->SimpleQueueType); + Py_CLEAR(state->EmptyError); + return 0; +} + +static void +queue_free(void *m) +{ + queue_clear((PyObject *)m); +} #include "clinic/_queuemodule.c.h" @@ -306,48 +352,26 @@ static PyMethodDef simplequeue_methods[] = { {NULL, NULL} /* sentinel */ }; +static struct PyMemberDef simplequeue_members[] = { + {"__weaklistoffset__", T_PYSSIZET, offsetof(simplequeueobject, weakreflist), READONLY}, + {NULL}, +}; -static PyTypeObject PySimpleQueueType = { - PyVarObject_HEAD_INIT(NULL, 0) - "_queue.SimpleQueue", /*tp_name*/ - sizeof(simplequeueobject), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)simplequeue_dealloc, /*tp_dealloc*/ - 0, /*tp_vectorcall_offset*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_as_async*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash*/ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE - | Py_TPFLAGS_HAVE_GC, /* tp_flags */ - simplequeue_new__doc__, /*tp_doc*/ - (traverseproc)simplequeue_traverse, /*tp_traverse*/ - 0, /*tp_clear*/ - 0, /*tp_richcompare*/ - offsetof(simplequeueobject, weakreflist), /*tp_weaklistoffset*/ - 0, /*tp_iter*/ - 0, /*tp_iternext*/ - simplequeue_methods, /*tp_methods*/ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - simplequeue_new /* tp_new */ +static PyType_Slot simplequeue_slots[] = { + {Py_tp_dealloc, simplequeue_dealloc}, + {Py_tp_doc, (void *)simplequeue_new__doc__}, + {Py_tp_traverse, simplequeue_traverse}, + {Py_tp_members, simplequeue_members}, + {Py_tp_methods, simplequeue_methods}, + {Py_tp_new, simplequeue_new}, + {0, NULL}, +}; + +static PyType_Spec simplequeue_spec = { + .name = "_queue.SimpleQueue", + .basicsize = sizeof(simplequeueobject), + .flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, + .slots = simplequeue_slots, }; @@ -358,15 +382,13 @@ PyDoc_STRVAR(queue_module_doc, This module is an implementation detail, please do not use it directly."); static struct PyModuleDef queuemodule = { - PyModuleDef_HEAD_INIT, - "_queue", - queue_module_doc, - -1, - NULL, - NULL, - NULL, - NULL, - NULL + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_queue", + .m_doc = queue_module_doc, + .m_size = sizeof(simplequeue_state), + .m_traverse = queue_traverse, + .m_clear = queue_clear, + .m_free = queue_free, }; @@ -374,26 +396,40 @@ PyMODINIT_FUNC PyInit__queue(void) { PyObject *m; + simplequeue_state *state; /* Create the module */ m = PyModule_Create(&queuemodule); if (m == NULL) return NULL; - EmptyError = PyErr_NewExceptionWithDoc( + state = simplequeue_get_state(m); + state->EmptyError = PyErr_NewExceptionWithDoc( "_queue.Empty", "Exception raised by Queue.get(block=0)/get_nowait().", NULL, NULL); - if (EmptyError == NULL) - return NULL; + if (state->EmptyError == NULL) + goto error; - Py_INCREF(EmptyError); - if (PyModule_AddObject(m, "Empty", EmptyError) < 0) - return NULL; + Py_INCREF(state->EmptyError); + if (PyModule_AddObject(m, "Empty", state->EmptyError) < 0) { + Py_DECREF(state->EmptyError); + goto error; + } - if (PyModule_AddType(m, &PySimpleQueueType) < 0) { - return NULL; + state->SimpleQueueType = (PyTypeObject *)PyType_FromModuleAndSpec(m, + &simplequeue_spec, + NULL); + if (state->SimpleQueueType == NULL) { + goto error; + } + if (PyModule_AddType(m, state->SimpleQueueType) < 0) { + goto error; } return m; + +error: + Py_DECREF(m); + return NULL; } diff --git a/Modules/clinic/_queuemodule.c.h b/Modules/clinic/_queuemodule.c.h index c25eacf08bc..8741f7d9aff 100644 --- a/Modules/clinic/_queuemodule.c.h +++ b/Modules/clinic/_queuemodule.c.h @@ -16,11 +16,11 @@ simplequeue_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { PyObject *return_value = NULL; - if ((type == &PySimpleQueueType) && + if ((type == simplequeue_get_state_by_type(type)->SimpleQueueType) && !_PyArg_NoPositional("SimpleQueue", args)) { goto exit; } - if ((type == &PySimpleQueueType) && + if ((type == simplequeue_get_state_by_type(type)->SimpleQueueType) && !_PyArg_NoKeywords("SimpleQueue", kwargs)) { goto exit; } @@ -133,42 +133,26 @@ PyDoc_STRVAR(_queue_SimpleQueue_get__doc__, "in that case)."); #define _QUEUE_SIMPLEQUEUE_GET_METHODDEF \ - {"get", (PyCFunction)(void(*)(void))_queue_SimpleQueue_get, METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_get__doc__}, + {"get", (PyCFunction)(void(*)(void))_queue_SimpleQueue_get, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_get__doc__}, static PyObject * -_queue_SimpleQueue_get_impl(simplequeueobject *self, int block, - PyObject *timeout); +_queue_SimpleQueue_get_impl(simplequeueobject *self, PyTypeObject *cls, + int block, PyObject *timeout); static PyObject * -_queue_SimpleQueue_get(simplequeueobject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +_queue_SimpleQueue_get(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; static const char * const _keywords[] = {"block", "timeout", NULL}; - static _PyArg_Parser _parser = {NULL, _keywords, "get", 0}; - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + static _PyArg_Parser _parser = {"|pO:get", _keywords, 0}; int block = 1; PyObject *timeout = Py_None; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, + &block, &timeout)) { goto exit; } - if (!noptargs) { - goto skip_optional_pos; - } - if (args[0]) { - block = PyObject_IsTrue(args[0]); - if (block < 0) { - goto exit; - } - if (!--noptargs) { - goto skip_optional_pos; - } - } - timeout = args[1]; -skip_optional_pos: - return_value = _queue_SimpleQueue_get_impl(self, block, timeout); + return_value = _queue_SimpleQueue_get_impl(self, cls, block, timeout); exit: return return_value; @@ -184,15 +168,27 @@ PyDoc_STRVAR(_queue_SimpleQueue_get_nowait__doc__, "raise the Empty exception."); #define _QUEUE_SIMPLEQUEUE_GET_NOWAIT_METHODDEF \ - {"get_nowait", (PyCFunction)_queue_SimpleQueue_get_nowait, METH_NOARGS, _queue_SimpleQueue_get_nowait__doc__}, + {"get_nowait", (PyCFunction)(void(*)(void))_queue_SimpleQueue_get_nowait, METH_METHOD|METH_FASTCALL|METH_KEYWORDS, _queue_SimpleQueue_get_nowait__doc__}, static PyObject * -_queue_SimpleQueue_get_nowait_impl(simplequeueobject *self); +_queue_SimpleQueue_get_nowait_impl(simplequeueobject *self, + PyTypeObject *cls); static PyObject * -_queue_SimpleQueue_get_nowait(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) +_queue_SimpleQueue_get_nowait(simplequeueobject *self, PyTypeObject *cls, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { - return _queue_SimpleQueue_get_nowait_impl(self); + PyObject *return_value = NULL; + static const char * const _keywords[] = { NULL}; + static _PyArg_Parser _parser = {":get_nowait", _keywords, 0}; + + if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser + )) { + goto exit; + } + return_value = _queue_SimpleQueue_get_nowait_impl(self, cls); + +exit: + return return_value; } PyDoc_STRVAR(_queue_SimpleQueue_empty__doc__, @@ -250,4 +246,4 @@ _queue_SimpleQueue_qsize(simplequeueobject *self, PyObject *Py_UNUSED(ignored)) exit: return return_value; } -/*[clinic end generated code: output=b4717e2974cbc909 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ce56b46fac150909 input=a9049054013a1b77]*/ From fd6f6fa403789c8877b1099cc6fcc437d2e54634 Mon Sep 17 00:00:00 2001 From: Don Kirkby Date: Sun, 8 Nov 2020 01:01:23 -0800 Subject: [PATCH 46/61] Minor wording change in concurrent.futures. (GH-23194) Fixes a grammar problem by adding a missing "as", and clarifies the wording of the valid ranges for max_workers. --- Doc/library/concurrent.futures.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst index 675a9ffdd07..61d6c1143cf 100644 --- a/Doc/library/concurrent.futures.rst +++ b/Doc/library/concurrent.futures.rst @@ -236,9 +236,9 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. An :class:`Executor` subclass that executes calls asynchronously using a pool of at most *max_workers* processes. If *max_workers* is ``None`` or not given, it will default to the number of processors on the machine. - If *max_workers* is lower or equal to ``0``, then a :exc:`ValueError` + If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` will be raised. - On Windows, *max_workers* must be equal or lower than ``61``. If it is not + On Windows, *max_workers* must be less than or equal to ``61``. If it is not then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then the default chosen will be at most ``61``, even if more processors are available. @@ -250,7 +250,7 @@ to a :class:`ProcessPoolExecutor` will result in deadlock. each worker process; *initargs* is a tuple of arguments passed to the initializer. Should *initializer* raise an exception, all currently pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, - as well any attempt to submit more jobs to the pool. + as well as any attempt to submit more jobs to the pool. .. versionchanged:: 3.3 When one of the worker processes terminates abruptly, a From 41761933c1c30bb6003b65eef1ba23a83db4eae4 Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 8 Nov 2020 10:05:27 +0100 Subject: [PATCH 47/61] bpo-41100: Support macOS 11 and Apple Silicon (GH-22855) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Lawrence D’Anna * Add support for macOS 11 and Apple Silicon (aka arm64) As a side effect of this work use the system copy of libffi on macOS, and remove the vendored copy * Support building on recent versions of macOS while deploying to older versions This allows building installers on macOS 11 while still supporting macOS 10.9. --- Lib/_osx_support.py | 44 +- Lib/ctypes/macholib/dyld.py | 12 + Lib/ctypes/test/test_macholib.py | 15 +- Lib/distutils/tests/test_build_ext.py | 2 +- Lib/test/test_bytes.py | 1 + Lib/test/test_platform.py | 2 +- Lib/test/test_posix.py | 228 +++++ Lib/test/test_time.py | 30 + Lib/test/test_unicode.py | 2 + Mac/BuildScript/build-installer.py | 30 +- Mac/BuildScript/openssl-mac-arm64.patch | 41 + Mac/README.rst | 37 + Mac/Tools/pythonw.c | 12 +- .../2020-11-01-16-40-23.bpo-41100.BApztP.rst | 8 + Modules/_ctypes/callbacks.c | 41 +- Modules/_ctypes/callproc.c | 127 ++- Modules/_ctypes/ctypes.h | 8 + Modules/_ctypes/malloc_closure.c | 15 +- Modules/getpath.c | 4 - Modules/posixmodule.c | 956 +++++++++++++----- Modules/timemodule.c | 214 +++- Python/bootstrap_hash.c | 42 +- Python/pytime.c | 31 +- configure | 45 +- configure.ac | 24 +- pyconfig.h.in | 3 + setup.py | 92 +- 27 files changed, 1654 insertions(+), 412 deletions(-) create mode 100644 Mac/BuildScript/openssl-mac-arm64.patch create mode 100644 Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst diff --git a/Lib/_osx_support.py b/Lib/_osx_support.py index e9efce7d7ed..8a696ee9895 100644 --- a/Lib/_osx_support.py +++ b/Lib/_osx_support.py @@ -110,6 +110,26 @@ def _get_system_version(): return _SYSTEM_VERSION +_SYSTEM_VERSION_TUPLE = None +def _get_system_version_tuple(): + """ + Return the macOS system version as a tuple + + The return value is safe to use to compare + two version numbers. + """ + global _SYSTEM_VERSION_TUPLE + if _SYSTEM_VERSION_TUPLE is None: + osx_version = _get_system_version() + if osx_version: + try: + _SYSTEM_VERSION_TUPLE = tuple(int(i) for i in osx_version.split('.')) + except ValueError: + _SYSTEM_VERSION_TUPLE = () + + return _SYSTEM_VERSION_TUPLE + + def _remove_original_values(_config_vars): """Remove original unmodified values for testing""" # This is needed for higher-level cross-platform tests of get_platform. @@ -132,14 +152,18 @@ def _supports_universal_builds(): # builds, in particular -isysroot and -arch arguments to the compiler. This # is in support of allowing 10.4 universal builds to run on 10.3.x systems. - osx_version = _get_system_version() - if osx_version: - try: - osx_version = tuple(int(i) for i in osx_version.split('.')) - except ValueError: - osx_version = '' + osx_version = _get_system_version_tuple() return bool(osx_version >= (10, 4)) if osx_version else False +def _supports_arm64_builds(): + """Returns True if arm64 builds are supported on this system""" + # There are two sets of systems supporting macOS/arm64 builds: + # 1. macOS 11 and later, unconditionally + # 2. macOS 10.15 with Xcode 12.2 or later + # For now the second category is ignored. + osx_version = _get_system_version_tuple() + return osx_version >= (11, 0) if osx_version else False + def _find_appropriate_compiler(_config_vars): """Find appropriate C compiler for extension module builds""" @@ -331,6 +355,12 @@ def compiler_fixup(compiler_so, cc_args): except ValueError: break + elif not _supports_arm64_builds(): + # Look for "-arch arm64" and drop that + for idx in range(len(compiler_so)): + if compiler_so[idx] == '-arch' and compiler_so[idx+1] == "arm64": + del compiler_so[idx:idx+2] + if 'ARCHFLAGS' in os.environ and not stripArch: # User specified different -arch flags in the environ, # see also distutils.sysconfig @@ -481,6 +511,8 @@ def get_platform_osx(_config_vars, osname, release, machine): if len(archs) == 1: machine = archs[0] + elif archs == ('arm64', 'x86_64'): + machine = 'universal2' elif archs == ('i386', 'ppc'): machine = 'fat' elif archs == ('i386', 'x86_64'): diff --git a/Lib/ctypes/macholib/dyld.py b/Lib/ctypes/macholib/dyld.py index 9d86b058765..1c3f8fd38b0 100644 --- a/Lib/ctypes/macholib/dyld.py +++ b/Lib/ctypes/macholib/dyld.py @@ -6,6 +6,11 @@ import os from ctypes.macholib.framework import framework_info from ctypes.macholib.dylib import dylib_info from itertools import * +try: + from _ctypes import _dyld_shared_cache_contains_path +except ImportError: + def _dyld_shared_cache_contains_path(*args): + raise NotImplementedError __all__ = [ 'dyld_find', 'framework_find', @@ -122,8 +127,15 @@ def dyld_find(name, executable_path=None, env=None): dyld_executable_path_search(name, executable_path), dyld_default_search(name, env), ), env): + if os.path.isfile(path): return path + try: + if _dyld_shared_cache_contains_path(path): + return path + except NotImplementedError: + pass + raise ValueError("dylib %s could not be found" % (name,)) def framework_find(fn, executable_path=None, env=None): diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/ctypes/test/test_macholib.py index 6b3526951ac..a1bac26a7df 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/ctypes/test/test_macholib.py @@ -45,19 +45,22 @@ def find_lib(name): class MachOTest(unittest.TestCase): @unittest.skipUnless(sys.platform == "darwin", 'OSX-specific test') def test_find(self): - - self.assertEqual(find_lib('pthread'), - '/usr/lib/libSystem.B.dylib') + # On Mac OS 11, system dylibs are only present in the shared cache, + # so symlinks like libpthread.dylib -> libSystem.B.dylib will not + # be resolved by dyld_find + self.assertIn(find_lib('pthread'), + ('/usr/lib/libSystem.B.dylib', '/usr/lib/libpthread.dylib')) result = find_lib('z') # Issue #21093: dyld default search path includes $HOME/lib and # /usr/local/lib before /usr/lib, which caused test failures if # a local copy of libz exists in one of them. Now ignore the head # of the path. - self.assertRegex(result, r".*/lib/libz\..*.*\.dylib") + self.assertRegex(result, r".*/lib/libz.*\.dylib") - self.assertEqual(find_lib('IOKit'), - '/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit') + self.assertIn(find_lib('IOKit'), + ('/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit', + '/System/Library/Frameworks/IOKit.framework/IOKit')) if __name__ == "__main__": unittest.main() diff --git a/Lib/distutils/tests/test_build_ext.py b/Lib/distutils/tests/test_build_ext.py index f9e0d766d87..6bb009a86f4 100644 --- a/Lib/distutils/tests/test_build_ext.py +++ b/Lib/distutils/tests/test_build_ext.py @@ -493,7 +493,7 @@ class BuildExtTestCase(TempdirManager, # format the target value as defined in the Apple # Availability Macros. We can't use the macro names since # at least one value we test with will not exist yet. - if target[1] < 10: + if target[:2] < (10, 10): # for 10.1 through 10.9.x -> "10n0" target = '%02d%01d0' % target else: diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index e61228d1a26..d550abfc656 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1036,6 +1036,7 @@ class BytesTest(BaseBytesTest, unittest.TestCase): c_char_p) PyBytes_FromFormat = pythonapi.PyBytes_FromFormat + PyBytes_FromFormat.argtypes = (c_char_p,) PyBytes_FromFormat.restype = py_object # basic tests diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index b5d21e54610..9b6d93cb5fc 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -246,7 +246,7 @@ class PlatformTest(unittest.TestCase): self.assertEqual(res[1], ('', '', '')) if sys.byteorder == 'little': - self.assertIn(res[2], ('i386', 'x86_64')) + self.assertIn(res[2], ('i386', 'x86_64', 'arm64')) else: self.assertEqual(res[2], 'PowerPC') diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index f57c88234b5..a522717751a 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1925,6 +1925,233 @@ class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): assert_python_ok(*args, PATH=path) +@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") +class TestPosixWeaklinking(unittest.TestCase): + # These test cases verify that weak linking support on macOS works + # as expected. These cases only test new behaviour introduced by weak linking, + # regular behaviour is tested by the normal test cases. + # + # See the section on Weak Linking in Mac/README.txt for more information. + def setUp(self): + import sysconfig + import platform + + config_vars = sysconfig.get_config_vars() + self.available = { nm for nm in config_vars if nm.startswith("HAVE_") and config_vars[nm] } + self.mac_ver = tuple(int(part) for part in platform.mac_ver()[0].split(".")) + + def _verify_available(self, name): + if name not in self.available: + raise unittest.SkipTest(f"{name} not weak-linked") + + def test_pwritev(self): + self._verify_available("HAVE_PWRITEV") + if self.mac_ver >= (10, 16): + self.assertTrue(hasattr(os, "pwritev"), "os.pwritev is not available") + self.assertTrue(hasattr(os, "preadv"), "os.readv is not available") + + else: + self.assertFalse(hasattr(os, "pwritev"), "os.pwritev is available") + self.assertFalse(hasattr(os, "preadv"), "os.readv is available") + + def test_stat(self): + self._verify_available("HAVE_FSTATAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FSTATAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FSTATAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.stat("file", dir_fd=0) + + def test_access(self): + self._verify_available("HAVE_FACCESSAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FACCESSAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FACCESSAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.access("file", os.R_OK, dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "follow_symlinks unavailable"): + os.access("file", os.R_OK, follow_symlinks=False) + + with self.assertRaisesRegex(NotImplementedError, "effective_ids unavailable"): + os.access("file", os.R_OK, effective_ids=True) + + def test_chmod(self): + self._verify_available("HAVE_FCHMODAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHMODAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FCHMODAT", posix._have_functions) + self.assertIn("HAVE_LCHMOD", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chmod("file", 0o644, dir_fd=0) + + def test_chown(self): + self._verify_available("HAVE_FCHOWNAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FCHOWNAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FCHOWNAT", posix._have_functions) + self.assertIn("HAVE_LCHOWN", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.chown("file", 0, 0, dir_fd=0) + + def test_link(self): + self._verify_available("HAVE_LINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_LINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_LINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "dst_dir_fd unavailable"): + os.link("source", "target", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd unavailable"): + os.link("source", "target", src_dir_fd=0, dst_dir_fd=0) + + # issue 41355: !HAVE_LINKAT code path ignores the follow_symlinks flag + with os_helper.temp_dir() as base_path: + link_path = os.path.join(base_path, "link") + target_path = os.path.join(base_path, "target") + source_path = os.path.join(base_path, "source") + + with open(source_path, "w") as fp: + fp.write("data") + + os.symlink("target", link_path) + + # Calling os.link should fail in the link(2) call, and + # should not reject *follow_symlinks* (to match the + # behaviour you'd get when building on a platform without + # linkat) + with self.assertRaises(FileExistsError): + os.link(source_path, link_path, follow_symlinks=True) + + with self.assertRaises(FileExistsError): + os.link(source_path, link_path, follow_symlinks=False) + + + def test_listdir_scandir(self): + self._verify_available("HAVE_FDOPENDIR") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_FDOPENDIR", posix._have_functions) + + else: + self.assertNotIn("HAVE_FDOPENDIR", posix._have_functions) + + with self.assertRaisesRegex(TypeError, "listdir: path should be string, bytes, os.PathLike or None, not int"): + os.listdir(0) + + with self.assertRaisesRegex(TypeError, "scandir: path should be string, bytes, os.PathLike or None, not int"): + os.scandir(0) + + def test_mkdir(self): + self._verify_available("HAVE_MKDIRAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_MKDIRAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_MKDIRAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.mkdir("dir", dir_fd=0) + + def test_rename_replace(self): + self._verify_available("HAVE_RENAMEAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_RENAMEAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_RENAMEAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.rename("a", "b", dst_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", src_dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "src_dir_fd and dst_dir_fd unavailable"): + os.replace("a", "b", dst_dir_fd=0) + + def test_unlink_rmdir(self): + self._verify_available("HAVE_UNLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_UNLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_UNLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.unlink("path", dir_fd=0) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.rmdir("path", dir_fd=0) + + def test_open(self): + self._verify_available("HAVE_OPENAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_OPENAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_OPENAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.open("path", os.O_RDONLY, dir_fd=0) + + def test_readlink(self): + self._verify_available("HAVE_READLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_READLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_READLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.readlink("path", dir_fd=0) + + def test_symlink(self): + self._verify_available("HAVE_SYMLINKAT") + if self.mac_ver >= (10, 10): + self.assertIn("HAVE_SYMLINKAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_SYMLINKAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.symlink("a", "b", dir_fd=0) + + def test_utime(self): + self._verify_available("HAVE_FUTIMENS") + self._verify_available("HAVE_UTIMENSAT") + if self.mac_ver >= (10, 13): + self.assertIn("HAVE_FUTIMENS", posix._have_functions) + self.assertIn("HAVE_UTIMENSAT", posix._have_functions) + + else: + self.assertNotIn("HAVE_FUTIMENS", posix._have_functions) + self.assertNotIn("HAVE_UTIMENSAT", posix._have_functions) + + with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): + os.utime("path", dir_fd=0) + + def test_main(): try: support.run_unittest( @@ -1932,6 +2159,7 @@ def test_main(): PosixGroupsTester, TestPosixSpawn, TestPosixSpawnP, + TestPosixWeaklinking ) finally: support.reap_children() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 6ced0470d07..32582986485 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -1041,6 +1041,36 @@ class TestOldPyTime(CPyTimeTestCase, unittest.TestCase): with self.assertRaises(ValueError): pytime_object_to_timespec(float('nan'), time_rnd) +@unittest.skipUnless(sys.platform == "darwin", "test weak linking on macOS") +class TestTimeWeaklinking(unittest.TestCase): + # These test cases verify that weak linking support on macOS works + # as expected. These cases only test new behaviour introduced by weak linking, + # regular behaviour is tested by the normal test cases. + # + # See the section on Weak Linking in Mac/README.txt for more information. + def test_clock_functions(self): + import sysconfig + import platform + + config_vars = sysconfig.get_config_vars() + var_name = "HAVE_CLOCK_GETTIME" + if var_name not in config_vars or not config_vars[var_name]: + raise unittest.SkipTest(f"{var_name} is not available") + + mac_ver = tuple(int(x) for x in platform.mac_ver()[0].split(".")) + + clock_names = [ + "CLOCK_MONOTONIC", "clock_gettime", "clock_gettime_ns", "clock_settime", + "clock_settime_ns", "clock_getres"] + + if mac_ver >= (10, 12): + for name in clock_names: + self.assertTrue(hasattr(time, name), f"time.{name} is not available") + + else: + for name in clock_names: + self.assertFalse(hasattr(time, name), f"time.{name} is available") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 90b09655822..4f5636e1426 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -2516,11 +2516,13 @@ class CAPITest(unittest.TestCase): def test_from_format(self): import_helper.import_module('ctypes') from ctypes import ( + c_char_p, pythonapi, py_object, sizeof, c_int, c_long, c_longlong, c_ssize_t, c_uint, c_ulong, c_ulonglong, c_size_t, c_void_p) name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) + _PyUnicode_FromFormat.argtypes = (c_char_p,) _PyUnicode_FromFormat.restype = py_object def PyUnicode_FromFormat(format, *args): diff --git a/Mac/BuildScript/build-installer.py b/Mac/BuildScript/build-installer.py index 2548b212d9e..0e76d3ca5bb 100755 --- a/Mac/BuildScript/build-installer.py +++ b/Mac/BuildScript/build-installer.py @@ -116,7 +116,8 @@ WORKDIR = "/tmp/_py" DEPSRC = os.path.join(WORKDIR, 'third-party') DEPSRC = os.path.expanduser('~/Universal/other-sources') -universal_opts_map = { '32-bit': ('i386', 'ppc',), +universal_opts_map = { 'universal2': ('arm64', 'x86_64'), + '32-bit': ('i386', 'ppc',), '64-bit': ('x86_64', 'ppc64',), 'intel': ('i386', 'x86_64'), 'intel-32': ('i386',), @@ -124,6 +125,7 @@ universal_opts_map = { '32-bit': ('i386', 'ppc',), '3-way': ('ppc', 'i386', 'x86_64'), 'all': ('i386', 'ppc', 'x86_64', 'ppc64',) } default_target_map = { + 'universal2': '10.9', '64-bit': '10.5', '3-way': '10.5', 'intel': '10.5', @@ -190,6 +192,27 @@ EXPECTED_SHARED_LIBS = {} def internalTk(): return getDeptargetTuple() >= (10, 6) + +def tweak_tcl_build(basedir, archList): + with open("Makefile", "r") as fp: + contents = fp.readlines() + + # For reasons I don't understand the tcl configure script + # decides that some stdlib symbols aren't present, before + # deciding that strtod is broken. + new_contents = [] + for line in contents: + if line.startswith("COMPAT_OBJS"): + # note: the space before strtod.o is intentional, + # the detection of a broken strtod results in + # "fixstrod.o" on this line. + for nm in ("strstr.o", "strtoul.o", " strtod.o"): + line = line.replace(nm, "") + new_contents.append(line) + + with open("Makefile", "w") as fp: + fp.writelines(new_contents) + # List of names of third party software built with this installer. # The names will be inserted into the rtf version of the License. THIRD_PARTY_LIBS = [] @@ -215,6 +238,9 @@ def library_recipes(): buildrecipe=build_universal_openssl, configure=None, install=None, + patches=[ + "openssl-mac-arm64.patch", + ], ), ]) @@ -231,6 +257,7 @@ def library_recipes(): '--libdir=/Library/Frameworks/Python.framework/Versions/%s/lib'%(getVersion(),), ], useLDFlags=False, + buildrecipe=tweak_tcl_build, install='make TCL_LIBRARY=%(TCL_LIBRARY)s && make install TCL_LIBRARY=%(TCL_LIBRARY)s DESTDIR=%(DESTDIR)s'%{ "DESTDIR": shellQuote(os.path.join(WORKDIR, 'libraries')), "TCL_LIBRARY": shellQuote('/Library/Frameworks/Python.framework/Versions/%s/lib/tcl8.6'%(getVersion())), @@ -801,6 +828,7 @@ def build_universal_openssl(basedir, archList): arch_opts = { "i386": ["darwin-i386-cc"], "x86_64": ["darwin64-x86_64-cc", "enable-ec_nistp_64_gcc_128"], + "arm64": ["darwin64-arm64-cc"], "ppc": ["darwin-ppc-cc"], "ppc64": ["darwin64-ppc-cc"], } diff --git a/Mac/BuildScript/openssl-mac-arm64.patch b/Mac/BuildScript/openssl-mac-arm64.patch new file mode 100644 index 00000000000..11267fb1187 --- /dev/null +++ b/Mac/BuildScript/openssl-mac-arm64.patch @@ -0,0 +1,41 @@ +diff -ur openssl-1.1.1g-orig/Configurations/10-main.conf openssl-1.1.1g/Configurations/10-main.conf +--- openssl-1.1.1g-orig/Configurations/10-main.conf 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/Configurations/10-main.conf 2020-07-26 12:21:32.000000000 +0200 +@@ -1557,6 +1557,14 @@ + bn_ops => "SIXTY_FOUR_BIT_LONG", + perlasm_scheme => "macosx", + }, ++ "darwin64-arm64-cc" => { ++ inherit_from => [ "darwin-common", asm("aarch64_asm") ], ++ CFLAGS => add("-Wall"), ++ cflags => add("-arch arm64"), ++ lib_cppflags => add("-DL_ENDIAN"), ++ bn_ops => "SIXTY_FOUR_BIT_LONG", ++ perlasm_scheme => "ios64", ++ }, + + ##### GNU Hurd + "hurd-x86" => { +diff -ur openssl-1.1.1g-orig/config openssl-1.1.1g/config +--- openssl-1.1.1g-orig/config 2020-04-21 14:22:39.000000000 +0200 ++++ openssl-1.1.1g/config 2020-07-26 12:21:59.000000000 +0200 +@@ -255,6 +255,9 @@ + ;; + x86_64) + echo "x86_64-apple-darwin${VERSION}" ++ ;; ++ arm64) ++ echo "arm64-apple-darwin${VERSION}" + ;; + *) + echo "i686-apple-darwin${VERSION}" +@@ -497,6 +500,9 @@ + else + OUT="darwin64-x86_64-cc" + fi ;; ++ x86_64-apple-darwin*) ++ OUT="darwin64-arm64-cc" ++ ;; + armv6+7-*-iphoneos) + __CNF_CFLAGS="$__CNF_CFLAGS -arch armv6 -arch armv7" + __CNF_CXXFLAGS="$__CNF_CXXFLAGS -arch armv6 -arch armv7" diff --git a/Mac/README.rst b/Mac/README.rst index ec7d873df27..f3638aa0019 100644 --- a/Mac/README.rst +++ b/Mac/README.rst @@ -120,6 +120,8 @@ support ppc (Xcode 4 on 10.6 and later systems). The flavor can be specified using the configure option ``--with-universal-archs=VALUE``. The following values are available: + * ``universal2``: ``arm64``, ``x86_64`` + * ``intel``: ``i386``, ``x86_64`` * ``intel-32``: ``i386`` @@ -155,6 +157,8 @@ following combinations of SDKs and universal-archs flavors are available: * 10.15 and later SDKs support ``intel-64`` only + * 11.0 and later SDKs support ``universal2`` + The makefile for a framework build will also install ``python3.x-32`` binaries when the universal architecture includes at least one 32-bit architecture (that is, for all flavors but ``64-bit`` and ``intel-64``). @@ -352,6 +356,39 @@ A framework install also installs some applications in ``/Applications/Python X. And lastly a framework installation installs files in ``/usr/local/bin``, all of them symbolic links to files in ``/Library/Frameworks/Python.framework/Versions/X.Y/bin``. +Weak linking support +==================== + +The CPython sources support building with the latest SDK while targetting deployment +to macOS 10.9. This is done through weak linking of symbols introduced in macOS +10.10 or later and checking for their availability at runtime. + +This requires the use of Apple's compiler toolchain on macOS 10.13 or later. + +The basic implementation pattern is: + +* ``HAVE_`` is a macro defined (or not) by the configure script + +* ``HAVE__RUNTIME`` is a macro defined in the relevant source + files. This expands to a call to ``__builtin_available`` when using + a new enough Apple compiler, and to a true value otherwise. + +* Use ``HAVE__RUNTIME`` before calling ````. This macro + *must* be used a the sole expression in an if statement:: + + if (HAVE__RUNTIME) { + /* is available */ + } + + Or: + + if (HAVE__RUNTIME) {} else { + /* is not available */ + } + + Using other patterns (such as ``!HAVE__RUNTIME``) is not supported + by Apple's compilers. + Resources ========= diff --git a/Mac/Tools/pythonw.c b/Mac/Tools/pythonw.c index c8bd3ba8d68..78813e818e7 100644 --- a/Mac/Tools/pythonw.c +++ b/Mac/Tools/pythonw.c @@ -95,9 +95,6 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) size_t count; cpu_type_t cpu_types[1]; short flags = 0; -#ifdef __LP64__ - int ch; -#endif if ((errno = posix_spawnattr_init(spawnattr)) != 0) { err(2, "posix_spawnattr_int"); @@ -119,10 +116,16 @@ setup_spawnattr(posix_spawnattr_t* spawnattr) #elif defined(__ppc__) cpu_types[0] = CPU_TYPE_POWERPC; + #elif defined(__i386__) cpu_types[0] = CPU_TYPE_X86; + +#elif defined(__arm64__) + cpu_types[0] = CPU_TYPE_ARM64; + #else # error "Unknown CPU" + #endif if (posix_spawnattr_setbinpref_np(spawnattr, count, @@ -220,7 +223,8 @@ main(int argc, char **argv) { /* We're weak-linking to posix-spawnv to ensure that * an executable build on 10.5 can work on 10.4. */ - if (posix_spawn != NULL) { + + if (&posix_spawn != NULL) { posix_spawnattr_t spawnattr = NULL; setup_spawnattr(&spawnattr); diff --git a/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst b/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst new file mode 100644 index 00000000000..6cbb279e762 --- /dev/null +++ b/Misc/NEWS.d/next/macOS/2020-11-01-16-40-23.bpo-41100.BApztP.rst @@ -0,0 +1,8 @@ +Add support for macOS 11 and Apple Silicon systems. + +It is now possible to build "Universal 2" binaries using +"--enable-universalsdk --with-universal-archs=universal2". + +Binaries build on later macOS versions can be deployed back to older +versions (tested up to macOS 10.9), when using the correct deployment +target. This is tested using Xcode 11 and later. diff --git a/Modules/_ctypes/callbacks.c b/Modules/_ctypes/callbacks.c index 5cd85772485..3686287e45a 100644 --- a/Modules/_ctypes/callbacks.c +++ b/Modules/_ctypes/callbacks.c @@ -1,6 +1,8 @@ #include "Python.h" #include "frameobject.h" +#include + #include #ifdef MS_WIN32 #include @@ -18,7 +20,7 @@ CThunkObject_dealloc(PyObject *myself) Py_XDECREF(self->callable); Py_XDECREF(self->restype); if (self->pcl_write) - ffi_closure_free(self->pcl_write); + Py_ffi_closure_free(self->pcl_write); PyObject_GC_Del(self); } @@ -362,8 +364,7 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, assert(CThunk_CheckExact((PyObject *)p)); - p->pcl_write = ffi_closure_alloc(sizeof(ffi_closure), - &p->pcl_exec); + p->pcl_write = Py_ffi_closure_alloc(sizeof(ffi_closure), &p->pcl_exec); if (p->pcl_write == NULL) { PyErr_NoMemory(); goto error; @@ -409,13 +410,35 @@ CThunkObject *_ctypes_alloc_callback(PyObject *callable, "ffi_prep_cif failed with %d", result); goto error; } -#if defined(X86_DARWIN) || defined(POWERPC_DARWIN) - result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); -#else - result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, - p, - p->pcl_exec); +#if HAVE_FFI_PREP_CLOSURE_LOC +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# else +# define HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME 1 +# endif + if (HAVE_FFI_PREP_CLOSURE_LOC_RUNTIME) { + result = ffi_prep_closure_loc(p->pcl_write, &p->cif, closure_fcn, + p, + p->pcl_exec); + } else #endif + { +#if USING_APPLE_OS_LIBFFI && defined(__arm64__) + PyErr_Format(PyExc_NotImplementedError, "ffi_prep_closure_loc() is missing"); + goto error; +#else +#ifdef MACOSX + #pragma clang diagnostic push + #pragma clang diagnostic ignored "-Wdeprecated-declarations" +#endif + result = ffi_prep_closure(p->pcl_write, &p->cif, closure_fcn, p); + +#ifdef MACOSX + #pragma clang diagnostic pop +#endif + +#endif + } if (result != FFI_OK) { PyErr_Format(PyExc_RuntimeError, "ffi_prep_closure failed with %d", result); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 261ae5ceb9e..a52d343031a 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -57,6 +57,8 @@ #include "Python.h" #include "structmember.h" // PyMemberDef +#include + #ifdef MS_WIN32 #include #include @@ -64,6 +66,10 @@ #include "ctypes_dlfcn.h" #endif +#ifdef __APPLE__ +#include +#endif + #ifdef MS_WIN32 #include #endif @@ -812,7 +818,8 @@ static int _call_function_pointer(int flags, ffi_type **atypes, ffi_type *restype, void *resmem, - int argcount) + int argcount, + int argtypecount) { PyThreadState *_save = NULL; /* For Py_BLOCK_THREADS and Py_UNBLOCK_THREADS */ PyObject *error_object = NULL; @@ -835,14 +842,70 @@ static int _call_function_pointer(int flags, if ((flags & FUNCFLAG_CDECL) == 0) cc = FFI_STDCALL; #endif - if (FFI_OK != ffi_prep_cif(&cif, - cc, - argcount, - restype, - atypes)) { - PyErr_SetString(PyExc_RuntimeError, - "ffi_prep_cif failed"); - return -1; + +# if USING_APPLE_OS_LIBFFI +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME __builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *) +# elif HAVE_FFI_PREP_CIF_VAR +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME true +# else +# define HAVE_FFI_PREP_CIF_VAR_RUNTIME false +# endif + + /* Even on Apple-arm64 the calling convention for variadic functions conincides + * with the standard calling convention in the case that the function called + * only with its fixed arguments. Thus, we do not need a special flag to be + * set on variadic functions. We treat a function as variadic if it is called + * with a nonzero number of variadic arguments */ + bool is_variadic = (argtypecount != 0 && argcount > argtypecount); + (void) is_variadic; + +#if defined(__APPLE__) && defined(__arm64__) + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + } else { + PyErr_SetString(PyExc_NotImplementedError, "ffi_prep_cif_var() is missing"); + return -1; + } + } +#endif + +#if HAVE_FFI_PREP_CIF_VAR + if (is_variadic) { + if (HAVE_FFI_PREP_CIF_VAR_RUNTIME) { + if (FFI_OK != ffi_prep_cif_var(&cif, + cc, + argtypecount, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif_var failed"); + return -1; + } + } else { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } + } + } else +#endif + + { + if (FFI_OK != ffi_prep_cif(&cif, + cc, + argcount, + restype, + atypes)) { + PyErr_SetString(PyExc_RuntimeError, + "ffi_prep_cif failed"); + return -1; + } } if (flags & (FUNCFLAG_USE_ERRNO | FUNCFLAG_USE_LASTERROR)) { @@ -1212,9 +1275,8 @@ PyObject *_ctypes_callproc(PPROC pProc, if (-1 == _call_function_pointer(flags, pProc, avalues, atypes, rtype, resbuf, - Py_SAFE_DOWNCAST(argcount, - Py_ssize_t, - int))) + Py_SAFE_DOWNCAST(argcount, Py_ssize_t, int), + Py_SAFE_DOWNCAST(argtype_count, Py_ssize_t, int))) goto cleanup; #ifdef WORDS_BIGENDIAN @@ -1398,6 +1460,42 @@ copy_com_pointer(PyObject *self, PyObject *args) } #else +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH +static PyObject *py_dyld_shared_cache_contains_path(PyObject *self, PyObject *args) +{ + PyObject *name, *name2; + char *name_str; + + if (__builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *)) { + int r; + + if (!PyArg_ParseTuple(args, "O", &name)) + return NULL; + + if (name == Py_None) + Py_RETURN_FALSE; + + if (PyUnicode_FSConverter(name, &name2) == 0) + return NULL; + name_str = PyBytes_AS_STRING(name2); + + r = _dyld_shared_cache_contains_path(name_str); + Py_DECREF(name2); + + if (r) { + Py_RETURN_TRUE; + } else { + Py_RETURN_FALSE; + } + + } else { + PyErr_SetString(PyExc_NotImplementedError, "_dyld_shared_cache_contains_path symbol is missing"); + return NULL; + } + + } +#endif + static PyObject *py_dl_open(PyObject *self, PyObject *args) { PyObject *name, *name2; @@ -1887,6 +1985,8 @@ buffer_info(PyObject *self, PyObject *arg) return Py_BuildValue("siN", dict->format, dict->ndim, shape); } + + PyMethodDef _ctypes_module_methods[] = { {"get_errno", get_errno, METH_NOARGS}, {"set_errno", set_errno, METH_VARARGS}, @@ -1908,6 +2008,9 @@ PyMethodDef _ctypes_module_methods[] = { "dlopen(name, flag={RTLD_GLOBAL|RTLD_LOCAL}) open a shared library"}, {"dlclose", py_dl_close, METH_VARARGS, "dlclose a library"}, {"dlsym", py_dl_sym, METH_VARARGS, "find symbol in shared library"}, +#endif +#ifdef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH + {"_dyld_shared_cache_contains_path", py_dyld_shared_cache_contains_path, METH_VARARGS, "check if path is in the shared cache"}, #endif {"alignment", align_func, METH_O, alignment_doc}, {"sizeof", sizeof_func, METH_O, sizeof_doc}, diff --git a/Modules/_ctypes/ctypes.h b/Modules/_ctypes/ctypes.h index 1effccf9cc5..3f20031d671 100644 --- a/Modules/_ctypes/ctypes.h +++ b/Modules/_ctypes/ctypes.h @@ -366,6 +366,14 @@ PyObject *_ctypes_get_errobj(int **pspace); extern PyObject *ComError; #endif +#if USING_MALLOC_CLOSURE_DOT_C +void Py_ffi_closure_free(void *p); +void *Py_ffi_closure_alloc(size_t size, void** codeloc); +#else +#define Py_ffi_closure_free ffi_closure_free +#define Py_ffi_closure_alloc ffi_closure_alloc +#endif + /* Local Variables: compile-command: "python setup.py -q build install --home ~" diff --git a/Modules/_ctypes/malloc_closure.c b/Modules/_ctypes/malloc_closure.c index f9cdb336958..4f220e42ff3 100644 --- a/Modules/_ctypes/malloc_closure.c +++ b/Modules/_ctypes/malloc_closure.c @@ -89,16 +89,27 @@ static void more_core(void) /******************************************************************/ /* put the item back into the free list */ -void ffi_closure_free(void *p) +void Py_ffi_closure_free(void *p) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + ffi_closure_free(p); + return; + } +#endif ITEM *item = (ITEM *)p; item->next = free_list; free_list = item; } /* return one item from the free list, allocating more if needed */ -void *ffi_closure_alloc(size_t ignored, void** codeloc) +void *Py_ffi_closure_alloc(size_t size, void** codeloc) { +#if USING_APPLE_OS_LIBFFI && HAVE_FFI_CLOSURE_ALLOC + if (__builtin_available(macos 10.15, ios 13, watchos 6, tvos 13, *)) { + return ffi_closure_alloc(size, codeloc); + } +#endif ITEM *item; if (!free_list) more_core(); diff --git a/Modules/getpath.c b/Modules/getpath.c index f7a6dd40443..44453f29df7 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -923,11 +923,7 @@ static PyStatus calculate_program_macos(wchar_t **abs_path_p) { char execpath[MAXPATHLEN + 1]; -#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_4 uint32_t nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#else - unsigned long nsexeclength = Py_ARRAY_LENGTH(execpath) - 1; -#endif /* On Mac OS X, if a script uses an interpreter of the form "#!/opt/python2.3/bin/python", the kernel only passes "python" diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 203f98515df..70b47c475fe 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -7,18 +7,6 @@ of the compiler used. Different compilers define their own feature test macro, e.g. '_MSC_VER'. */ -#ifdef __APPLE__ - /* - * Step 1 of support for weak-linking a number of symbols existing on - * OSX 10.4 and later, see the comment in the #ifdef __APPLE__ block - * at the end of this file for more information. - */ -# pragma weak lchown -# pragma weak statvfs -# pragma weak fstatvfs - -#endif /* __APPLE__ */ - #define PY_SSIZE_T_CLEAN #include "Python.h" @@ -55,6 +43,127 @@ #include /* needed for ctermid() */ +/* + * A number of APIs are available on macOS from a certain macOS version. + * To support building with a new SDK while deploying to older versions + * the availability test is split into two: + * - HAVE_: The configure check for compile time availability + * - HAVE__RUNTIME: Runtime check for availability + * + * The latter is always true when not on macOS, or when using a compiler + * that does not support __has_builtin (older versions of Xcode). + * + * Due to compiler restrictions there is one valid use of HAVE__RUNTIME: + * if (HAVE__RUNTIME) { ... } + * + * In mixing the test with other tests or using negations will result in compile + * errors. + */ +#if defined(__APPLE__) + +#if defined(__has_builtin) && __has_builtin(__builtin_available) +# define HAVE_FSTATAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FACCESSAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FCHMODAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FCHOWNAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_LINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FDOPENDIR_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_MKDIRAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_RENAMEAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_UNLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_OPENAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_READLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_SYMLINKAT_RUNTIME __builtin_available(macOS 10.10, iOS 8.0, *) +# define HAVE_FUTIMENS_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) +# define HAVE_UTIMENSAT_RUNTIME __builtin_available(macOS 10.13, iOS 11.0, tvOS 11.0, watchOS 4.0, *) +# define HAVE_PWRITEV_RUNTIME __builtin_available(macOS 11.0, iOS 14.0, tvOS 14.0, watchOS 7.0, *) + +# define HAVE_POSIX_SPAWN_SETSID_RUNTIME __builtin_available(macOS 10.15, *) + +#else /* Xcode 8 or earlier */ + + /* __builtin_available is not present in these compilers, but + * some of the symbols might be weak linked (10.10 SDK or later + * deploying on 10.9. + * + * Fall back to the older style of availability checking for + * symbols introduced in macOS 10.10. + */ + +# ifdef HAVE_FSTATAT +# define HAVE_FSTATAT_RUNTIME (fstatat != NULL) +# endif + +# ifdef HAVE_FACCESSAT +# define HAVE_FACCESSAT_RUNTIME (faccessat != NULL) +# endif + +# ifdef HAVE_FCHMODAT +# define HAVE_FCHMODAT_RUNTIME (fchmodat != NULL) +# endif + +# ifdef HAVE_FCHOWNAT +# define HAVE_FCHOWNAT_RUNTIME (fchownat != NULL) +# endif + +# ifdef HAVE_LINKAT +# define HAVE_LINKAT_RUNTIME (linkat != NULL) +# endif + +# ifdef HAVE_FDOPENDIR +# define HAVE_FDOPENDIR_RUNTIME (fdopendir != NULL) +# endif + +# ifdef HAVE_MKDIRAT +# define HAVE_MKDIRAT_RUNTIME (mkdirat != NULL) +# endif + +# ifdef HAVE_RENAMEAT +# define HAVE_RENAMEAT_RUNTIME (renameat != NULL) +# endif + +# ifdef HAVE_UNLINKAT +# define HAVE_UNLINKAT_RUNTIME (unlinkat != NULL) +# endif + +# ifdef HAVE_OPENAT +# define HAVE_OPENAT_RUNTIME (openat != NULL) +# endif + +# ifdef HAVE_READLINKAT +# define HAVE_READLINKAT_RUNTIME (readlinkat != NULL) +# endif + +# ifdef HAVE_SYMLINKAT +# define HAVE_SYMLINKAT_RUNTIME (symlinkat != NULL) +# endif + +#endif + +#ifdef HAVE_FUTIMESAT +/* Some of the logic for weak linking depends on this assertion */ +# error "HAVE_FUTIMESAT unexpectedly defined" +#endif + +#else +# define HAVE_FSTATAT_RUNTIME 1 +# define HAVE_FACCESSAT_RUNTIME 1 +# define HAVE_FCHMODAT_RUNTIME 1 +# define HAVE_FCHOWNAT_RUNTIME 1 +# define HAVE_LINKAT_RUNTIME 1 +# define HAVE_FDOPENDIR_RUNTIME 1 +# define HAVE_MKDIRAT_RUNTIME 1 +# define HAVE_RENAMEAT_RUNTIME 1 +# define HAVE_UNLINKAT_RUNTIME 1 +# define HAVE_OPENAT_RUNTIME 1 +# define HAVE_READLINKAT_RUNTIME 1 +# define HAVE_SYMLINKAT_RUNTIME 1 +# define HAVE_FUTIMENS_RUNTIME 1 +# define HAVE_UTIMENSAT_RUNTIME 1 +# define HAVE_PWRITEV_RUNTIME 1 +#endif + + #ifdef __cplusplus extern "C" { #endif @@ -2360,6 +2469,10 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, STRUCT_STAT st; int result; +#ifdef HAVE_FSTATAT + int fstatat_unavailable = 0; +#endif + #if !defined(MS_WINDOWS) && !defined(HAVE_FSTATAT) && !defined(HAVE_LSTAT) if (follow_symlinks_specified(function_name, follow_symlinks)) return NULL; @@ -2386,15 +2499,27 @@ posix_do_stat(PyObject *module, const char *function_name, path_t *path, else #endif /* HAVE_LSTAT */ #ifdef HAVE_FSTATAT - if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) - result = fstatat(dir_fd, path->narrow, &st, + if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { + if (HAVE_FSTATAT_RUNTIME) { + result = fstatat(dir_fd, path->narrow, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - else + + } else { + fstatat_unavailable = 1; + } + } else #endif /* HAVE_FSTATAT */ result = STAT(path->narrow, &st); #endif /* MS_WINDOWS */ Py_END_ALLOW_THREADS +#ifdef HAVE_FSTATAT + if (fstatat_unavailable) { + argument_unavailable_error("stat", "dir_fd"); + return NULL; + } +#endif + if (result != 0) { return path_error(path); } @@ -2808,6 +2933,10 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, int result; #endif +#ifdef HAVE_FACCESSAT + int faccessat_unavailable = 0; +#endif + #ifndef HAVE_FACCESSAT if (follow_symlinks_specified("access", follow_symlinks)) return -1; @@ -2842,17 +2971,40 @@ os_access_impl(PyObject *module, path_t *path, int mode, int dir_fd, if ((dir_fd != DEFAULT_DIR_FD) || effective_ids || !follow_symlinks) { - int flags = 0; - if (!follow_symlinks) - flags |= AT_SYMLINK_NOFOLLOW; - if (effective_ids) - flags |= AT_EACCESS; - result = faccessat(dir_fd, path->narrow, mode, flags); + + if (HAVE_FACCESSAT_RUNTIME) { + int flags = 0; + if (!follow_symlinks) + flags |= AT_SYMLINK_NOFOLLOW; + if (effective_ids) + flags |= AT_EACCESS; + result = faccessat(dir_fd, path->narrow, mode, flags); + } else { + faccessat_unavailable = 1; + } } else #endif result = access(path->narrow, mode); Py_END_ALLOW_THREADS + +#ifdef HAVE_FACCESSAT + if (faccessat_unavailable) { + if (dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("access", "dir_fd"); + return -1; + } + if (follow_symlinks_specified("access", follow_symlinks)) + return -1; + + if (effective_ids) { + argument_unavailable_error("access", "effective_ids"); + return -1; + } + /* should be unreachable */ + return -1; + } +#endif return_value = !result; #endif @@ -3050,6 +3202,7 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, #ifdef HAVE_FCHMODAT int fchmodat_nofollow_unsupported = 0; + int fchmodat_unsupported = 0; #endif #if !(defined(HAVE_FCHMODAT) || defined(HAVE_LCHMOD)) @@ -3085,42 +3238,56 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, if (path->fd != -1) result = fchmod(path->fd, mode); else -#endif +#endif /* HAVE_CHMOD */ #ifdef HAVE_LCHMOD if ((!follow_symlinks) && (dir_fd == DEFAULT_DIR_FD)) result = lchmod(path->narrow, mode); else -#endif +#endif /* HAVE_LCHMOD */ #ifdef HAVE_FCHMODAT if ((dir_fd != DEFAULT_DIR_FD) || !follow_symlinks) { - /* - * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! - * The documentation specifically shows how to use it, - * and then says it isn't implemented yet. - * (true on linux with glibc 2.15, and openindiana 3.x) - * - * Once it is supported, os.chmod will automatically - * support dir_fd and follow_symlinks=False. (Hopefully.) - * Until then, we need to be careful what exception we raise. - */ - result = fchmodat(dir_fd, path->narrow, mode, - follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - /* - * But wait! We can't throw the exception without allowing threads, - * and we can't do that in this nested scope. (Macro trickery, sigh.) - */ - fchmodat_nofollow_unsupported = - result && - ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && - !follow_symlinks; + if (HAVE_FCHMODAT_RUNTIME) { + /* + * fchmodat() doesn't currently support AT_SYMLINK_NOFOLLOW! + * The documentation specifically shows how to use it, + * and then says it isn't implemented yet. + * (true on linux with glibc 2.15, and openindiana 3.x) + * + * Once it is supported, os.chmod will automatically + * support dir_fd and follow_symlinks=False. (Hopefully.) + * Until then, we need to be careful what exception we raise. + */ + result = fchmodat(dir_fd, path->narrow, mode, + follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); + /* + * But wait! We can't throw the exception without allowing threads, + * and we can't do that in this nested scope. (Macro trickery, sigh.) + */ + fchmodat_nofollow_unsupported = + result && + ((errno == ENOTSUP) || (errno == EOPNOTSUPP)) && + !follow_symlinks; + } else { + fchmodat_unsupported = 1; + fchmodat_nofollow_unsupported = 1; + + result = -1; + } } else -#endif +#endif /* HAVE_FHCMODAT */ result = chmod(path->narrow, mode); Py_END_ALLOW_THREADS if (result) { #ifdef HAVE_FCHMODAT + if (fchmodat_unsupported) { + if (dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("chmod", "dir_fd"); + return NULL; + } + } + if (fchmodat_nofollow_unsupported) { if (dir_fd != DEFAULT_DIR_FD) dir_fd_and_follow_symlinks_invalid("chmod", @@ -3130,10 +3297,10 @@ os_chmod_impl(PyObject *module, path_t *path, int mode, int dir_fd, return NULL; } else -#endif +#endif /* HAVE_FCHMODAT */ return path_error(path); } -#endif +#endif /* MS_WINDOWS */ Py_RETURN_NONE; } @@ -3421,6 +3588,10 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, { int result; +#if defined(HAVE_FCHOWNAT) + int fchownat_unsupported = 0; +#endif + #if !(defined(HAVE_LCHOWN) || defined(HAVE_FCHOWNAT)) if (follow_symlinks_specified("chown", follow_symlinks)) return NULL; @@ -3429,19 +3600,6 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, fd_and_follow_symlinks_invalid("chown", path->fd, follow_symlinks)) return NULL; -#ifdef __APPLE__ - /* - * This is for Mac OS X 10.3, which doesn't have lchown. - * (But we still have an lchown symbol because of weak-linking.) - * It doesn't have fchownat either. So there's no possibility - * of a graceful failover. - */ - if ((!follow_symlinks) && (lchown == NULL)) { - follow_symlinks_specified("chown", follow_symlinks); - return NULL; - } -#endif - if (PySys_Audit("os.chown", "OIIi", path->object, uid, gid, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { return NULL; @@ -3459,14 +3617,28 @@ os_chown_impl(PyObject *module, path_t *path, uid_t uid, gid_t gid, else #endif #ifdef HAVE_FCHOWNAT - if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) + if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { + if (HAVE_FCHOWNAT_RUNTIME) { result = fchownat(dir_fd, path->narrow, uid, gid, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); - else + } else { + fchownat_unsupported = 1; + } + } else #endif result = chown(path->narrow, uid, gid); Py_END_ALLOW_THREADS +#ifdef HAVE_FCHOWNAT + if (fchownat_unsupported) { + /* This would be incorrect if the current platform + * doesn't support lchown. + */ + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -3712,6 +3884,9 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #else int result; #endif +#if defined(HAVE_LINKAT) + int linkat_unavailable = 0; +#endif #ifndef HAVE_LINKAT if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD)) { @@ -3746,15 +3921,43 @@ os_link_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, #ifdef HAVE_LINKAT if ((src_dir_fd != DEFAULT_DIR_FD) || (dst_dir_fd != DEFAULT_DIR_FD) || - (!follow_symlinks)) - result = linkat(src_dir_fd, src->narrow, - dst_dir_fd, dst->narrow, - follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + (!follow_symlinks)) { + + if (HAVE_LINKAT_RUNTIME) { + + result = linkat(src_dir_fd, src->narrow, + dst_dir_fd, dst->narrow, + follow_symlinks ? AT_SYMLINK_FOLLOW : 0); + + } +#ifdef __APPLE__ + else { + if (src_dir_fd == DEFAULT_DIR_FD && dst_dir_fd == DEFAULT_DIR_FD) { + /* See issue 41355: This matches the behaviour of !HAVE_LINKAT */ + result = link(src->narrow, dst->narrow); + } else { + linkat_unavailable = 1; + } + } +#endif + } else #endif /* HAVE_LINKAT */ result = link(src->narrow, dst->narrow); Py_END_ALLOW_THREADS +#ifdef HAVE_LINKAT + if (linkat_unavailable) { + /* Either or both dir_fd arguments were specified */ + if (src_dir_fd != DEFAULT_DIR_FD) { + argument_unavailable_error("link", "src_dir_fd"); + } else { + argument_unavailable_error("link", "dst_dir_fd"); + } + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif /* MS_WINDOWS */ @@ -3877,6 +4080,7 @@ _posix_listdir(path_t *path, PyObject *list) errno = 0; #ifdef HAVE_FDOPENDIR if (path->fd != -1) { + if (HAVE_FDOPENDIR_RUNTIME) { /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(path->fd); if (fd == -1) @@ -3887,6 +4091,11 @@ _posix_listdir(path_t *path, PyObject *list) Py_BEGIN_ALLOW_THREADS dirp = fdopendir(fd); Py_END_ALLOW_THREADS + } else { + PyErr_SetString(PyExc_TypeError, + "listdir: path should be string, bytes, os.PathLike or None, not int"); + return NULL; + } } else #endif @@ -4200,6 +4409,9 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) /*[clinic end generated code: output=a70446903abe821f input=e965f68377e9b1ce]*/ { int result; +#ifdef HAVE_MKDIRAT + int mkdirat_unavailable = 0; +#endif if (PySys_Audit("os.mkdir", "Oii", path->object, mode, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4216,9 +4428,14 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) #else Py_BEGIN_ALLOW_THREADS #if HAVE_MKDIRAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_MKDIRAT_RUNTIME) { result = mkdirat(dir_fd, path->narrow, mode); - else + + } else { + mkdirat_unavailable = 1; + } + } else #endif #if defined(__WATCOMC__) && !defined(__QNX__) result = mkdir(path->narrow); @@ -4226,6 +4443,14 @@ os_mkdir_impl(PyObject *module, path_t *path, int mode, int dir_fd) result = mkdir(path->narrow, mode); #endif Py_END_ALLOW_THREADS + +#if HAVE_MKDIRAT + if (mkdirat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result < 0) return path_error(path); #endif /* MS_WINDOWS */ @@ -4335,6 +4560,10 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is const char *function_name = is_replace ? "replace" : "rename"; int dir_fd_specified; +#ifdef HAVE_RENAMEAT + int renameat_unavailable = 0; +#endif + #ifdef MS_WINDOWS BOOL result; int flags = is_replace ? MOVEFILE_REPLACE_EXISTING : 0; @@ -4374,13 +4603,25 @@ internal_rename(path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd, int is Py_BEGIN_ALLOW_THREADS #ifdef HAVE_RENAMEAT - if (dir_fd_specified) - result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); - else + if (dir_fd_specified) { + if (HAVE_RENAMEAT_RUNTIME) { + result = renameat(src_dir_fd, src->narrow, dst_dir_fd, dst->narrow); + } else { + renameat_unavailable = 1; + } + } else #endif result = rename(src->narrow, dst->narrow); Py_END_ALLOW_THREADS + +#ifdef HAVE_RENAMEAT + if (renameat_unavailable) { + argument_unavailable_error(function_name, "src_dir_fd and dst_dir_fd"); + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif @@ -4456,6 +4697,9 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) /*[clinic end generated code: output=080eb54f506e8301 input=38c8b375ca34a7e2]*/ { int result; +#ifdef HAVE_UNLINKAT + int unlinkat_unavailable = 0; +#endif if (PySys_Audit("os.rmdir", "Oi", path->object, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4468,14 +4712,26 @@ os_rmdir_impl(PyObject *module, path_t *path, int dir_fd) result = !RemoveDirectoryW(path->wide); #else #ifdef HAVE_UNLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_UNLINKAT_RUNTIME) { result = unlinkat(dir_fd, path->narrow, AT_REMOVEDIR); - else + } else { + unlinkat_unavailable = 1; + result = -1; + } + } else #endif result = rmdir(path->narrow); #endif Py_END_ALLOW_THREADS +#ifdef HAVE_UNLINKAT + if (unlinkat_unavailable) { + argument_unavailable_error("rmdir", "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -4619,6 +4875,9 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) /*[clinic end generated code: output=621797807b9963b1 input=d7bcde2b1b2a2552]*/ { int result; +#ifdef HAVE_UNLINKAT + int unlinkat_unavailable = 0; +#endif if (PySys_Audit("os.remove", "Oi", path->object, dir_fd == DEFAULT_DIR_FD ? -1 : dir_fd) < 0) { @@ -4632,15 +4891,27 @@ os_unlink_impl(PyObject *module, path_t *path, int dir_fd) result = !Py_DeleteFileW(path->wide); #else #ifdef HAVE_UNLINKAT - if (dir_fd != DEFAULT_DIR_FD) + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_UNLINKAT_RUNTIME) { + result = unlinkat(dir_fd, path->narrow, 0); - else + } else { + unlinkat_unavailable = 1; + } + } else #endif /* HAVE_UNLINKAT */ result = unlink(path->narrow); #endif _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS +#ifdef HAVE_UNLINKAT + if (unlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error(path); @@ -4811,7 +5082,16 @@ typedef struct { static int utime_dir_fd(utime_t *ut, int dir_fd, const char *path, int follow_symlinks) { -#ifdef HAVE_UTIMENSAT +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + if (HAVE_UTIMENSAT_RUNTIME) { + int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; + UTIME_TO_TIMESPEC; + return utimensat(dir_fd, path, time, flags); + } else { + errno = ENOSYS; + return -1; + } +#elif defined(HAVE_UTIMENSAT) int flags = follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW; UTIME_TO_TIMESPEC; return utimensat(dir_fd, path, time, flags); @@ -4838,11 +5118,30 @@ static int utime_fd(utime_t *ut, int fd) { #ifdef HAVE_FUTIMENS + + if (HAVE_FUTIMENS_RUNTIME) { + UTIME_TO_TIMESPEC; return futimens(fd, time); -#else + + } else +#ifndef HAVE_FUTIMES + { + /* Not sure if this can happen */ + PyErr_SetString( + PyExc_RuntimeError, + "neither futimens nor futimes are supported" + " on this system"); + return -1; + } +#endif + +#endif +#ifdef HAVE_FUTIMES + { UTIME_TO_TIMEVAL; return futimes(fd, time); + } #endif } @@ -4861,11 +5160,27 @@ static int utime_nofollow_symlinks(utime_t *ut, const char *path) { #ifdef HAVE_UTIMENSAT - UTIME_TO_TIMESPEC; - return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); -#else + if (HAVE_UTIMENSAT_RUNTIME) { + UTIME_TO_TIMESPEC; + return utimensat(DEFAULT_DIR_FD, path, time, AT_SYMLINK_NOFOLLOW); + } else +#ifndef HAVE_LUTIMES + { + /* Not sure if this can happen */ + PyErr_SetString( + PyExc_RuntimeError, + "neither utimensat nor lutimes are supported" + " on this system"); + return -1; + } +#endif +#endif + +#ifdef HAVE_LUTIMES + { UTIME_TO_TIMEVAL; return lutimes(path, time); + } #endif } @@ -4876,7 +5191,15 @@ utime_nofollow_symlinks(utime_t *ut, const char *path) static int utime_default(utime_t *ut, const char *path) { -#ifdef HAVE_UTIMENSAT +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + if (HAVE_UTIMENSAT_RUNTIME) { + UTIME_TO_TIMESPEC; + return utimensat(DEFAULT_DIR_FD, path, time, 0); + } else { + UTIME_TO_TIMEVAL; + return utimes(path, time); + } +#elif defined(HAVE_UTIMENSAT) UTIME_TO_TIMESPEC; return utimensat(DEFAULT_DIR_FD, path, time, 0); #elif defined(HAVE_UTIMES) @@ -5085,9 +5408,10 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, #endif #if defined(HAVE_FUTIMESAT) || defined(HAVE_UTIMENSAT) - if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) + if ((dir_fd != DEFAULT_DIR_FD) || (!follow_symlinks)) { result = utime_dir_fd(&utime, dir_fd, path->narrow, follow_symlinks); - else + + } else #endif #if defined(HAVE_FUTIMES) || defined(HAVE_FUTIMENS) @@ -5100,6 +5424,14 @@ os_utime_impl(PyObject *module, path_t *path, PyObject *times, PyObject *ns, Py_END_ALLOW_THREADS +#if defined(__APPLE__) && defined(HAVE_UTIMENSAT) + /* See utime_dir_fd implementation */ + if (result == -1 && errno == ENOSYS) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result < 0) { /* see previous comment about not putting filename in error here */ posix_error(); @@ -5498,6 +5830,9 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg } if (setsid) { +#ifdef HAVE_POSIX_SPAWN_SETSID_RUNTIME + if (HAVE_POSIX_SPAWN_SETSID_RUNTIME) { +#endif #ifdef POSIX_SPAWN_SETSID all_flags |= POSIX_SPAWN_SETSID; #elif defined(POSIX_SPAWN_SETSID_NP) @@ -5506,6 +5841,14 @@ parse_posix_spawn_flags(PyObject *module, const char *func_name, PyObject *setpg argument_unavailable_error(func_name, "setsid"); return -1; #endif + +#ifdef HAVE_POSIX_SPAWN_SETSID_RUNTIME + } else { + argument_unavailable_error(func_name, "setsid"); + return -1; + } +#endif /* HAVE_POSIX_SPAWN_SETSID_RUNTIME */ + } if (setsigmask) { @@ -8115,16 +8458,30 @@ os_readlink_impl(PyObject *module, path_t *path, int dir_fd) #if defined(HAVE_READLINK) char buffer[MAXPATHLEN+1]; ssize_t length; +#ifdef HAVE_READLINKAT + int readlinkat_unavailable = 0; +#endif Py_BEGIN_ALLOW_THREADS #ifdef HAVE_READLINKAT - if (dir_fd != DEFAULT_DIR_FD) - length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); - else + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_READLINKAT_RUNTIME) { + length = readlinkat(dir_fd, path->narrow, buffer, MAXPATHLEN); + } else { + readlinkat_unavailable = 1; + } + } else #endif length = readlink(path->narrow, buffer, MAXPATHLEN); Py_END_ALLOW_THREADS +#ifdef HAVE_READLINKAT + if (readlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (length < 0) { return path_error(path); } @@ -8320,6 +8677,9 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, static int windows_has_symlink_unprivileged_flag = TRUE; #else int result; +#ifdef HAVE_SYMLINKAT + int symlinkat_unavailable = 0; +#endif #endif if (PySys_Audit("os.symlink", "OOi", src->object, dst->object, @@ -8382,14 +8742,25 @@ os_symlink_impl(PyObject *module, path_t *src, path_t *dst, } Py_BEGIN_ALLOW_THREADS -#if HAVE_SYMLINKAT - if (dir_fd != DEFAULT_DIR_FD) - result = symlinkat(src->narrow, dir_fd, dst->narrow); - else +#ifdef HAVE_SYMLINKAT + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_SYMLINKAT_RUNTIME) { + result = symlinkat(src->narrow, dir_fd, dst->narrow); + } else { + symlinkat_unavailable = 1; + } + } else #endif result = symlink(src->narrow, dst->narrow); Py_END_ALLOW_THREADS +#ifdef HAVE_SYMLINKAT + if (symlinkat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return NULL; + } +#endif + if (result) return path_error2(src, dst); #endif @@ -8660,6 +9031,9 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) { int fd; int async_err = 0; +#ifdef HAVE_OPENAT + int openat_unavailable = 0; +#endif #ifdef O_CLOEXEC int *atomic_flag_works = &_Py_open_cloexec_works; @@ -8684,9 +9058,15 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) fd = _wopen(path->wide, flags, mode); #else #ifdef HAVE_OPENAT - if (dir_fd != DEFAULT_DIR_FD) - fd = openat(dir_fd, path->narrow, flags, mode); - else + if (dir_fd != DEFAULT_DIR_FD) { + if (HAVE_OPENAT_RUNTIME) { + fd = openat(dir_fd, path->narrow, flags, mode); + + } else { + openat_unavailable = 1; + fd = -1; + } + } else #endif /* HAVE_OPENAT */ fd = open(path->narrow, flags, mode); #endif /* !MS_WINDOWS */ @@ -8694,6 +9074,13 @@ os_open_impl(PyObject *module, path_t *path, int flags, int mode, int dir_fd) } while (fd < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); _Py_END_SUPPRESS_IPH +#ifdef HAVE_OPENAT + if (openat_unavailable) { + argument_unavailable_error(NULL, "dir_fd"); + return -1; + } +#endif + if (fd < 0) { if (!async_err) PyErr_SetFromErrnoWithFilenameObject(PyExc_OSError, path->object); @@ -9233,12 +9620,25 @@ os_preadv_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else do { +#ifdef __APPLE__ +/* This entire function will be removed from the module dict when the API + * is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +#endif Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH n = preadv(fd, iov, cnt, offset); _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (n < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif iov_cleanup(iov, buf, cnt); @@ -9846,6 +10246,15 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); #else + +#ifdef __APPLE__ +/* This entire function will be removed from the module dict when the API + * is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#pragma clang diagnostic ignored "-Wunguarded-availability-new" +#endif do { Py_BEGIN_ALLOW_THREADS _Py_BEGIN_SUPPRESS_IPH @@ -9853,6 +10262,11 @@ os_pwritev_impl(PyObject *module, int fd, PyObject *buffers, Py_off_t offset, _Py_END_SUPPRESS_IPH Py_END_ALLOW_THREADS } while (result < 0 && errno == EINTR && !(async_err = PyErr_CheckSignals())); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif iov_cleanup(iov, buf, cnt); @@ -10742,13 +11156,6 @@ os_statvfs_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS #ifdef HAVE_FSTATVFS if (path->fd != -1) { -#ifdef __APPLE__ - /* handle weak-linking on Mac OS X 10.3 */ - if (fstatvfs == NULL) { - fd_specified("statvfs", path->fd); - return NULL; - } -#endif result = fstatvfs(path->fd, &st); } else @@ -12832,13 +13239,17 @@ _Py_COMP_DIAG_POP const char *path = PyBytes_AS_STRING(ub); if (self->dir_fd != DEFAULT_DIR_FD) { #ifdef HAVE_FSTATAT + if (HAVE_FSTATAT_RUNTIME) { result = fstatat(self->dir_fd, path, &st, follow_symlinks ? 0 : AT_SYMLINK_NOFOLLOW); -#else + } else + +#endif /* HAVE_FSTATAT */ + { Py_DECREF(ub); PyErr_SetString(PyExc_NotImplementedError, "can't fetch stat"); return NULL; -#endif /* HAVE_FSTATAT */ + } } else #endif @@ -13637,6 +14048,7 @@ os_scandir_impl(PyObject *module, path_t *path) errno = 0; #ifdef HAVE_FDOPENDIR if (iterator->path.fd != -1) { + if (HAVE_FDOPENDIR_RUNTIME) { /* closedir() closes the FD, so we duplicate it */ fd = _Py_dup(iterator->path.fd); if (fd == -1) @@ -13645,6 +14057,11 @@ os_scandir_impl(PyObject *module, path_t *path) Py_BEGIN_ALLOW_THREADS iterator->dirp = fdopendir(fd); Py_END_ALLOW_THREADS + } else { + PyErr_SetString(PyExc_TypeError, + "scandir: path should be string, bytes, os.PathLike or None, not int"); + return NULL; + } } else #endif @@ -14713,137 +15130,210 @@ all_ins(PyObject *m) } -static const char * const have_functions[] = { + +#define PROBE(name, test) \ + static int name(void) \ + { \ + if (test) { \ + return 1; \ + } else { \ + return 0; \ + } \ + } + +#ifdef HAVE_FSTATAT +PROBE(probe_fstatat, HAVE_FSTATAT_RUNTIME) +#endif #ifdef HAVE_FACCESSAT - "HAVE_FACCESSAT", -#endif - -#ifdef HAVE_FCHDIR - "HAVE_FCHDIR", -#endif - -#ifdef HAVE_FCHMOD - "HAVE_FCHMOD", +PROBE(probe_faccessat, HAVE_FACCESSAT_RUNTIME) #endif #ifdef HAVE_FCHMODAT - "HAVE_FCHMODAT", -#endif - -#ifdef HAVE_FCHOWN - "HAVE_FCHOWN", +PROBE(probe_fchmodat, HAVE_FCHMODAT_RUNTIME) #endif #ifdef HAVE_FCHOWNAT - "HAVE_FCHOWNAT", -#endif - -#ifdef HAVE_FEXECVE - "HAVE_FEXECVE", -#endif - -#ifdef HAVE_FDOPENDIR - "HAVE_FDOPENDIR", -#endif - -#ifdef HAVE_FPATHCONF - "HAVE_FPATHCONF", -#endif - -#ifdef HAVE_FSTATAT - "HAVE_FSTATAT", -#endif - -#ifdef HAVE_FSTATVFS - "HAVE_FSTATVFS", -#endif - -#if defined HAVE_FTRUNCATE || defined MS_WINDOWS - "HAVE_FTRUNCATE", -#endif - -#ifdef HAVE_FUTIMENS - "HAVE_FUTIMENS", -#endif - -#ifdef HAVE_FUTIMES - "HAVE_FUTIMES", -#endif - -#ifdef HAVE_FUTIMESAT - "HAVE_FUTIMESAT", +PROBE(probe_fchownat, HAVE_FCHOWNAT_RUNTIME) #endif #ifdef HAVE_LINKAT - "HAVE_LINKAT", +PROBE(probe_linkat, HAVE_LINKAT_RUNTIME) #endif -#ifdef HAVE_LCHFLAGS - "HAVE_LCHFLAGS", -#endif - -#ifdef HAVE_LCHMOD - "HAVE_LCHMOD", -#endif - -#ifdef HAVE_LCHOWN - "HAVE_LCHOWN", -#endif - -#ifdef HAVE_LSTAT - "HAVE_LSTAT", -#endif - -#ifdef HAVE_LUTIMES - "HAVE_LUTIMES", -#endif - -#ifdef HAVE_MEMFD_CREATE - "HAVE_MEMFD_CREATE", +#ifdef HAVE_FDOPENDIR +PROBE(probe_fdopendir, HAVE_FDOPENDIR_RUNTIME) #endif #ifdef HAVE_MKDIRAT - "HAVE_MKDIRAT", -#endif - -#ifdef HAVE_MKFIFOAT - "HAVE_MKFIFOAT", -#endif - -#ifdef HAVE_MKNODAT - "HAVE_MKNODAT", -#endif - -#ifdef HAVE_OPENAT - "HAVE_OPENAT", -#endif - -#ifdef HAVE_READLINKAT - "HAVE_READLINKAT", +PROBE(probe_mkdirat, HAVE_MKDIRAT_RUNTIME) #endif #ifdef HAVE_RENAMEAT - "HAVE_RENAMEAT", -#endif - -#ifdef HAVE_SYMLINKAT - "HAVE_SYMLINKAT", +PROBE(probe_renameat, HAVE_RENAMEAT_RUNTIME) #endif #ifdef HAVE_UNLINKAT - "HAVE_UNLINKAT", +PROBE(probe_unlinkat, HAVE_UNLINKAT_RUNTIME) +#endif + +#ifdef HAVE_OPENAT +PROBE(probe_openat, HAVE_OPENAT_RUNTIME) +#endif + +#ifdef HAVE_READLINKAT +PROBE(probe_readlinkat, HAVE_READLINKAT_RUNTIME) +#endif + +#ifdef HAVE_SYMLINKAT +PROBE(probe_symlinkat, HAVE_SYMLINKAT_RUNTIME) +#endif + +#ifdef HAVE_FUTIMENS +PROBE(probe_futimens, HAVE_FUTIMENS_RUNTIME) #endif #ifdef HAVE_UTIMENSAT - "HAVE_UTIMENSAT", +PROBE(probe_utimensat, HAVE_UTIMENSAT_RUNTIME) +#endif + + + + +static const struct have_function { + const char * const label; + int (*probe)(void); +} have_functions[] = { + +#ifdef HAVE_FACCESSAT + { "HAVE_FACCESSAT", probe_faccessat }, +#endif + +#ifdef HAVE_FCHDIR + { "HAVE_FCHDIR", NULL }, +#endif + +#ifdef HAVE_FCHMOD + { "HAVE_FCHMOD", NULL }, +#endif + +#ifdef HAVE_FCHMODAT + { "HAVE_FCHMODAT", probe_fchmodat }, +#endif + +#ifdef HAVE_FCHOWN + { "HAVE_FCHOWN", NULL }, +#endif + +#ifdef HAVE_FCHOWNAT + { "HAVE_FCHOWNAT", probe_fchownat }, +#endif + +#ifdef HAVE_FEXECVE + { "HAVE_FEXECVE", NULL }, +#endif + +#ifdef HAVE_FDOPENDIR + { "HAVE_FDOPENDIR", probe_fdopendir }, +#endif + +#ifdef HAVE_FPATHCONF + { "HAVE_FPATHCONF", NULL }, +#endif + +#ifdef HAVE_FSTATAT + { "HAVE_FSTATAT", probe_fstatat }, +#endif + +#ifdef HAVE_FSTATVFS + { "HAVE_FSTATVFS", NULL }, +#endif + +#if defined HAVE_FTRUNCATE || defined MS_WINDOWS + { "HAVE_FTRUNCATE", NULL }, +#endif + +#ifdef HAVE_FUTIMENS + { "HAVE_FUTIMENS", probe_futimens }, +#endif + +#ifdef HAVE_FUTIMES + { "HAVE_FUTIMES", NULL }, +#endif + +#ifdef HAVE_FUTIMESAT + { "HAVE_FUTIMESAT", NULL }, +#endif + +#ifdef HAVE_LINKAT + { "HAVE_LINKAT", probe_linkat }, +#endif + +#ifdef HAVE_LCHFLAGS + { "HAVE_LCHFLAGS", NULL }, +#endif + +#ifdef HAVE_LCHMOD + { "HAVE_LCHMOD", NULL }, +#endif + +#ifdef HAVE_LCHOWN + { "HAVE_LCHOWN", NULL }, +#endif + +#ifdef HAVE_LSTAT + { "HAVE_LSTAT", NULL }, +#endif + +#ifdef HAVE_LUTIMES + { "HAVE_LUTIMES", NULL }, +#endif + +#ifdef HAVE_MEMFD_CREATE + { "HAVE_MEMFD_CREATE", NULL }, +#endif + +#ifdef HAVE_MKDIRAT + { "HAVE_MKDIRAT", probe_mkdirat }, +#endif + +#ifdef HAVE_MKFIFOAT + { "HAVE_MKFIFOAT", NULL }, +#endif + +#ifdef HAVE_MKNODAT + { "HAVE_MKNODAT", NULL }, +#endif + +#ifdef HAVE_OPENAT + { "HAVE_OPENAT", probe_openat }, +#endif + +#ifdef HAVE_READLINKAT + { "HAVE_READLINKAT", probe_readlinkat }, +#endif + +#ifdef HAVE_RENAMEAT + { "HAVE_RENAMEAT", probe_renameat }, +#endif + +#ifdef HAVE_SYMLINKAT + { "HAVE_SYMLINKAT", probe_symlinkat }, +#endif + +#ifdef HAVE_UNLINKAT + { "HAVE_UNLINKAT", probe_unlinkat }, +#endif + +#ifdef HAVE_UTIMENSAT + { "HAVE_UTIMENSAT", probe_utimensat }, #endif #ifdef MS_WINDOWS - "MS_WINDOWS", + { "MS_WINDOWS", NULL }, #endif - NULL + { NULL, NULL } }; @@ -14852,6 +15342,23 @@ posixmodule_exec(PyObject *m) { _posixstate *state = get_posix_state(m); +#if defined(HAVE_PWRITEV) + if (HAVE_PWRITEV_RUNTIME) {} else { + PyObject* dct = PyModule_GetDict(m); + + if (dct == NULL) { + return -1; + } + + if (PyDict_DelItemString(dct, "pwritev") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "preadv") == -1) { + PyErr_Clear(); + } + } +#endif + /* Initialize environ dictionary */ PyObject *v = convertenviron(); Py_XINCREF(v); @@ -14964,44 +15471,6 @@ posixmodule_exec(PyObject *m) PyModule_AddObject(m, "uname_result", (PyObject *)UnameResultType); state->UnameResultType = (PyObject *)UnameResultType; -#ifdef __APPLE__ - /* - * Step 2 of weak-linking support on Mac OS X. - * - * The code below removes functions that are not available on the - * currently active platform. - * - * This block allow one to use a python binary that was build on - * OSX 10.4 on OSX 10.3, without losing access to new APIs on - * OSX 10.4. - */ -#ifdef HAVE_FSTATVFS - if (fstatvfs == NULL) { - if (PyObject_DelAttrString(m, "fstatvfs") == -1) { - return -1; - } - } -#endif /* HAVE_FSTATVFS */ - -#ifdef HAVE_STATVFS - if (statvfs == NULL) { - if (PyObject_DelAttrString(m, "statvfs") == -1) { - return -1; - } - } -#endif /* HAVE_STATVFS */ - -# ifdef HAVE_LCHOWN - if (lchown == NULL) { - if (PyObject_DelAttrString(m, "lchown") == -1) { - return -1; - } - } -#endif /* HAVE_LCHOWN */ - - -#endif /* __APPLE__ */ - if ((state->billion = PyLong_FromLong(1000000000)) == NULL) return -1; #if defined(HAVE_WAIT3) || defined(HAVE_WAIT4) @@ -15031,14 +15500,17 @@ posixmodule_exec(PyObject *m) if (!list) { return -1; } - for (const char * const *trace = have_functions; *trace; trace++) { - PyObject *unicode = PyUnicode_DecodeASCII(*trace, strlen(*trace), NULL); + for (const struct have_function *trace = have_functions; trace->label; trace++) { + PyObject *unicode; + if (trace->probe && !trace->probe()) continue; + unicode = PyUnicode_DecodeASCII(trace->label, strlen(trace->label), NULL); if (!unicode) return -1; if (PyList_Append(list, unicode)) return -1; Py_DECREF(unicode); } + PyModule_AddObject(m, "_have_functions", list); return 0; diff --git a/Modules/timemodule.c b/Modules/timemodule.c index eb192c5e7fd..80eab30c95d 100644 --- a/Modules/timemodule.c +++ b/Modules/timemodule.c @@ -51,6 +51,15 @@ #define _Py_tzname tzname #endif +#if defined(__APPLE__ ) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +# endif +#endif +#ifndef HAVE_CLOCK_GETTIME_RUNTIME +# define HAVE_CLOCK_GETTIME_RUNTIME 1 +#endif + #define SEC_TO_NS (1000 * 1000 * 1000) /* Forward declarations */ @@ -149,6 +158,16 @@ perf_counter(_Py_clock_info_t *info) } #ifdef HAVE_CLOCK_GETTIME + +#ifdef __APPLE__ +/* + * The clock_* functions will be removed from the module + * dict entirely when the C API is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#endif + static PyObject * time_clock_gettime(PyObject *self, PyObject *args) { @@ -297,6 +316,11 @@ PyDoc_STRVAR(clock_getres_doc, "clock_getres(clk_id) -> floating point number\n\ \n\ Return the resolution (precision) of the specified clock clk_id."); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif /* HAVE_CLOCK_GETRES */ #ifdef HAVE_PTHREAD_GETCPUCLOCKID @@ -1162,31 +1186,35 @@ _PyTime_GetProcessTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #if defined(HAVE_CLOCK_GETTIME) \ && (defined(CLOCK_PROCESS_CPUTIME_ID) || defined(CLOCK_PROF)) struct timespec ts; + + if (HAVE_CLOCK_GETTIME_RUNTIME) { + #ifdef CLOCK_PROF - const clockid_t clk_id = CLOCK_PROF; - const char *function = "clock_gettime(CLOCK_PROF)"; + const clockid_t clk_id = CLOCK_PROF; + const char *function = "clock_gettime(CLOCK_PROF)"; #else - const clockid_t clk_id = CLOCK_PROCESS_CPUTIME_ID; - const char *function = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)"; + const clockid_t clk_id = CLOCK_PROCESS_CPUTIME_ID; + const char *function = "clock_gettime(CLOCK_PROCESS_CPUTIME_ID)"; #endif - if (clock_gettime(clk_id, &ts) == 0) { - if (info) { - struct timespec res; - info->implementation = function; - info->monotonic = 1; - info->adjustable = 0; - if (clock_getres(clk_id, &res)) { - PyErr_SetFromErrno(PyExc_OSError); + if (clock_gettime(clk_id, &ts) == 0) { + if (info) { + struct timespec res; + info->implementation = function; + info->monotonic = 1; + info->adjustable = 0; + if (clock_getres(clk_id, &res)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } + info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + } + + if (_PyTime_FromTimespec(tp, &ts) < 0) { return -1; } - info->resolution = res.tv_sec + res.tv_nsec * 1e-9; + return 0; } - - if (_PyTime_FromTimespec(tp, &ts) < 0) { - return -1; - } - return 0; } #endif @@ -1390,6 +1418,16 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #elif defined(HAVE_CLOCK_GETTIME) && defined(CLOCK_PROCESS_CPUTIME_ID) #define HAVE_THREAD_TIME + +#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +static int +_PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) + __attribute__((availability(macos, introduced=10.12))) + __attribute__((availability(ios, introduced=10.0))) + __attribute__((availability(tvos, introduced=10.0))) + __attribute__((availability(watchos, introduced=3.0))); +#endif + static int _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) { @@ -1421,6 +1459,15 @@ _PyTime_GetThreadTimeWithInfo(_PyTime_t *tp, _Py_clock_info_t *info) #endif #ifdef HAVE_THREAD_TIME +#ifdef __APPLE__ +/* + * The clock_* functions will be removed from the module + * dict entirely when the C API is not available. + */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wunguarded-availability" +#endif + static PyObject * time_thread_time(PyObject *self, PyObject *unused) { @@ -1451,6 +1498,11 @@ PyDoc_STRVAR(thread_time_ns_doc, \n\ Thread time for profiling as nanoseconds:\n\ sum of the kernel and user-space CPU time."); + +#ifdef __APPLE__ +#pragma clang diagnostic pop +#endif + #endif @@ -1500,9 +1552,19 @@ time_get_clock_info(PyObject *self, PyObject *args) } #ifdef HAVE_THREAD_TIME else if (strcmp(name, "thread_time") == 0) { - if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + +#ifdef __APPLE__ + if (HAVE_CLOCK_GETTIME_RUNTIME) { +#endif + if (_PyTime_GetThreadTimeWithInfo(&t, &info) < 0) { + return NULL; + } +#ifdef __APPLE__ + } else { + PyErr_SetString(PyExc_ValueError, "unknown clock"); return NULL; } +#endif } #endif else { @@ -1783,68 +1845,116 @@ if it is -1, mktime() should guess based on the date and time.\n"); static int time_exec(PyObject *module) { +#if defined(__APPLE__) && defined(HAVE_CLOCK_GETTIME) + if (HAVE_CLOCK_GETTIME_RUNTIME) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + if (dct == NULL) { + return -1; + } + + if (PyDict_DelItemString(dct, "clock_gettime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_gettime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_settime_ns") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "clock_getres") == -1) { + PyErr_Clear(); + } + } +#endif +#if defined(__APPLE__) && defined(HAVE_THREAD_TIME) + if (HAVE_CLOCK_GETTIME_RUNTIME) { + /* pass: ^^^ cannot use '!' here */ + } else { + PyObject* dct = PyModule_GetDict(module); + + if (PyDict_DelItemString(dct, "thread_time") == -1) { + PyErr_Clear(); + } + if (PyDict_DelItemString(dct, "thread_time_ns") == -1) { + PyErr_Clear(); + } + } +#endif /* Set, or reset, module variables like time.timezone */ if (init_timezone(module) < 0) { return -1; } #if defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) + if (HAVE_CLOCK_GETTIME_RUNTIME) { #ifdef CLOCK_REALTIME - if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_REALTIME) < 0) { + return -1; + } #endif + #ifdef CLOCK_MONOTONIC - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { - return -1; - } + + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC) < 0) { + return -1; + } + #endif #ifdef CLOCK_MONOTONIC_RAW - if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_MONOTONIC_RAW) < 0) { + return -1; + } #endif + #ifdef CLOCK_HIGHRES - if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_HIGHRES) < 0) { + return -1; + } #endif #ifdef CLOCK_PROCESS_CPUTIME_ID - if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_PROCESS_CPUTIME_ID) < 0) { + return -1; + } #endif + #ifdef CLOCK_THREAD_CPUTIME_ID - if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_THREAD_CPUTIME_ID) < 0) { + return -1; + } #endif #ifdef CLOCK_PROF - if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_PROF) < 0) { + return -1; + } #endif #ifdef CLOCK_BOOTTIME - if (PyModule_AddIntMacro(module, CLOCK_BOOTTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_BOOTTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_TAI - if (PyModule_AddIntMacro(module, CLOCK_TAI) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_TAI) < 0) { + return -1; + } #endif #ifdef CLOCK_UPTIME - if (PyModule_AddIntMacro(module, CLOCK_UPTIME) < 0) { - return -1; - } + if (PyModule_AddIntMacro(module, CLOCK_UPTIME) < 0) { + return -1; + } #endif #ifdef CLOCK_UPTIME_RAW - if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { - return -1; - } + + if (PyModule_AddIntMacro(module, CLOCK_UPTIME_RAW) < 0) { + return -1; + } #endif + } #endif /* defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_CLOCK_GETRES) */ diff --git a/Python/bootstrap_hash.c b/Python/bootstrap_hash.c index 47369305ee8..a212f69870e 100644 --- a/Python/bootstrap_hash.c +++ b/Python/bootstrap_hash.c @@ -25,6 +25,16 @@ # include #endif +#if defined(__APPLE__) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME __builtin_available(macOS 10.12, iOS 10.10, tvOS 10.0, watchOS 3.0, *) +# endif +#endif +#ifndef HAVE_GETENTRYPY_GETRANDOM_RUNTIME +# define HAVE_GETENTRYPY_GETRANDOM_RUNTIME 1 +#endif + + #ifdef Py_DEBUG int _Py_HashSecret_Initialized = 0; #else @@ -208,6 +218,16 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) error. getentropy() is retried if it failed with EINTR: interrupted by a signal. */ + +#if defined(__APPLE__) && defined(__has_attribute) && __has_attribute(availability) +static int +py_getentropy(char *buffer, Py_ssize_t size, int raise) + __attribute__((availability(macos,introduced=10.12))) + __attribute__((availability(ios,introduced=10.0))) + __attribute__((availability(tvos,introduced=10.0))) + __attribute__((availability(watchos,introduced=3.0))); +#endif + static int py_getentropy(char *buffer, Py_ssize_t size, int raise) { @@ -498,19 +518,21 @@ pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise) #else #if defined(PY_GETRANDOM) || defined(PY_GETENTROPY) + if (HAVE_GETENTRYPY_GETRANDOM_RUNTIME) { #ifdef PY_GETRANDOM - res = py_getrandom(buffer, size, blocking, raise); + res = py_getrandom(buffer, size, blocking, raise); #else - res = py_getentropy(buffer, size, raise); + res = py_getentropy(buffer, size, raise); #endif - if (res < 0) { - return -1; - } - if (res == 1) { - return 0; - } - /* getrandom() or getentropy() function is not available: failed with - ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ + if (res < 0) { + return -1; + } + if (res == 1) { + return 0; + } + /* getrandom() or getentropy() function is not available: failed with + ENOSYS or EPERM. Fall back on reading from /dev/urandom. */ + } /* end of availability block */ #endif return dev_urandom(buffer, size, raise); diff --git a/Python/pytime.c b/Python/pytime.c index b121b432f42..89d63e08042 100644 --- a/Python/pytime.c +++ b/Python/pytime.c @@ -5,6 +5,12 @@ #if defined(__APPLE__) #include /* mach_absolute_time(), mach_timebase_info() */ + +#if defined(__APPLE__) && defined(__has_builtin) +# if __has_builtin(__builtin_available) +# define HAVE_CLOCK_GETTIME_RUNTIME __builtin_available(macOS 10.12, iOS 10.0, tvOS 10.0, watchOS 3.0, *) +# endif +#endif #endif #define _PyTime_check_mul_overflow(a, b) \ @@ -683,15 +689,22 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) #else /* MS_WINDOWS */ int err; -#ifdef HAVE_CLOCK_GETTIME +#if defined(HAVE_CLOCK_GETTIME) struct timespec ts; -#else +#endif + +#if !defined(HAVE_CLOCK_GETTIME) || defined(__APPLE__) struct timeval tv; #endif assert(info == NULL || raise); #ifdef HAVE_CLOCK_GETTIME + +#ifdef HAVE_CLOCK_GETTIME_RUNTIME + if (HAVE_CLOCK_GETTIME_RUNTIME) { +#endif + err = clock_gettime(CLOCK_REALTIME, &ts); if (err) { if (raise) { @@ -715,7 +728,14 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->resolution = 1e-9; } } -#else /* HAVE_CLOCK_GETTIME */ + +#ifdef HAVE_CLOCK_GETTIME_RUNTIME + } else { +#endif + +#endif + +#if !defined(HAVE_CLOCK_GETTIME) || defined(HAVE_CLOCK_GETTIME_RUNTIME) /* test gettimeofday() */ err = gettimeofday(&tv, (struct timezone *)NULL); @@ -735,6 +755,11 @@ pygettimeofday(_PyTime_t *tp, _Py_clock_info_t *info, int raise) info->monotonic = 0; info->adjustable = 1; } + +#if defined(HAVE_CLOCK_GETTIME_RUNTIME) && defined(HAVE_CLOCK_GETTIME) + } /* end of availibity block */ +#endif + #endif /* !HAVE_CLOCK_GETTIME */ #endif /* !MS_WINDOWS */ return 0; diff --git a/configure b/configure index bc87485bf51..68d692d0f67 100755 --- a/configure +++ b/configure @@ -1511,8 +1511,8 @@ Optional Packages: specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", - "intel-64", or "all") see Mac/README.rst + ("universal2", "32-bit", "64-bit", "3-way", "intel", + "intel-32", "intel-64", or "all") see Mac/README.rst --with-framework-name=FRAMEWORK specify the name for the python framework on macOS only valid when --enable-framework is set. see @@ -7002,7 +7002,7 @@ fi -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= @@ -7529,6 +7529,11 @@ $as_echo "$CC" >&6; } LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -7550,7 +7555,7 @@ $as_echo "$CC" >&6; } ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - as_fn_error $? "proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way" "$LINENO" 5 + as_fn_error $? "proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way" "$LINENO" 5 ;; esac @@ -9382,7 +9387,7 @@ fi MACOSX_DEFAULT_ARCH="ppc" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac else @@ -9392,9 +9397,12 @@ fi ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - as_fn_error $? "Unexpected output of 'arch' on OSX" "$LINENO" 5 + as_fn_error $? "Unexpected output of 'arch' on macOS" "$LINENO" 5 ;; esac @@ -12029,6 +12037,31 @@ else { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 $as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for _dyld_shared_cache_contains_path" >&5 +$as_echo_n "checking for _dyld_shared_cache_contains_path... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +#include +int +main () +{ +void *x=_dyld_shared_cache_contains_path + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH 1" >>confdefs.h + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext diff --git a/configure.ac b/configure.ac index 49ed09a3a00..1edafc342b7 100644 --- a/configure.ac +++ b/configure.ac @@ -218,7 +218,7 @@ AC_ARG_WITH(universal-archs, AS_HELP_STRING([--with-universal-archs=ARCH], [specify the kind of universal binary that should be created. this option is only valid when --enable-universalsdk is set; options are: - ("32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") + ("universal2", "32-bit", "64-bit", "3-way", "intel", "intel-32", "intel-64", or "all") see Mac/README.rst]), [ UNIVERSAL_ARCHS="$withval" @@ -1597,7 +1597,7 @@ AC_SUBST(BASECFLAGS) AC_SUBST(CFLAGS_NODIST) AC_SUBST(LDFLAGS_NODIST) -# The -arch flags for universal builds on OSX +# The -arch flags for universal builds on macOS UNIVERSAL_ARCH_FLAGS= AC_SUBST(UNIVERSAL_ARCH_FLAGS) @@ -1898,6 +1898,11 @@ yes) LIPO_32BIT_FLAGS="-extract ppc7400 -extract i386" ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; + universal2) + UNIVERSAL_ARCH_FLAGS="-arch arm64 -arch x86_64" + LIPO_32BIT_FLAGS="" + ARCH_RUN_32BIT="true" + ;; intel) UNIVERSAL_ARCH_FLAGS="-arch i386 -arch x86_64" LIPO_32BIT_FLAGS="-extract i386" @@ -1919,7 +1924,7 @@ yes) ARCH_RUN_32BIT="/usr/bin/arch -i386 -ppc" ;; *) - AC_MSG_ERROR([proper usage is --with-universal-arch=32-bit|64-bit|all|intel|3-way]) + AC_MSG_ERROR([proper usage is --with-universal-arch=universal2|32-bit|64-bit|all|intel|3-way]) ;; esac @@ -2489,7 +2494,7 @@ case $ac_sys_system/$ac_sys_release in MACOSX_DEFAULT_ARCH="ppc" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac else @@ -2499,9 +2504,12 @@ case $ac_sys_system/$ac_sys_release in ;; ppc) MACOSX_DEFAULT_ARCH="ppc64" + ;; + arm64) + MACOSX_DEFAULT_ARCH="arm64" ;; *) - AC_MSG_ERROR([Unexpected output of 'arch' on OSX]) + AC_MSG_ERROR([Unexpected output of 'arch' on macOS]) ;; esac @@ -3774,6 +3782,12 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no) ]) +AC_MSG_CHECKING(for _dyld_shared_cache_contains_path) +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[#include ]], [[void *x=_dyld_shared_cache_contains_path]])], + [AC_DEFINE(HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH, 1, Define if you have the '_dyld_shared_cache_contains_path' function.) + AC_MSG_RESULT(yes)], + [AC_MSG_RESULT(no) +]) AC_MSG_CHECKING(for memfd_create) AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ diff --git a/pyconfig.h.in b/pyconfig.h.in index af8a3d6d9c3..d71ad3fdc86 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -784,6 +784,9 @@ /* Define if you have the 'prlimit' functions. */ #undef HAVE_PRLIMIT +/* Define if you have the '_dyld_shared_cache_contains_path' function. */ +#undef HAVE_DYLD_SHARED_CACHE_CONTAINS_PATH + /* Define to 1 if you have the header file. */ #undef HAVE_PROCESS_H diff --git a/setup.py b/setup.py index b3f47603f7a..c12b5f50425 100644 --- a/setup.py +++ b/setup.py @@ -239,6 +239,13 @@ def is_macosx_sdk_path(path): or path.startswith('/Library/') ) +def grep_headers_for(function, headers): + for header in headers: + with open(header, 'r') as f: + if function in f.read(): + return True + return False + def find_file(filename, std_dirs, paths): """Searches for the directory where a given file is located, and returns a possibly-empty list of additional directories, or None @@ -2105,43 +2112,17 @@ class PyBuildExt(build_ext): library_dirs=added_lib_dirs)) return True - def configure_ctypes_darwin(self, ext): - # Darwin (OS X) uses preconfigured files, in - # the Modules/_ctypes/libffi_osx directory. - ffi_srcdir = os.path.abspath(os.path.join(self.srcdir, 'Modules', - '_ctypes', 'libffi_osx')) - sources = [os.path.join(ffi_srcdir, p) - for p in ['ffi.c', - 'x86/darwin64.S', - 'x86/x86-darwin.S', - 'x86/x86-ffi_darwin.c', - 'x86/x86-ffi64.c', - 'powerpc/ppc-darwin.S', - 'powerpc/ppc-darwin_closure.S', - 'powerpc/ppc-ffi_darwin.c', - 'powerpc/ppc64-darwin_closure.S', - ]] - - # Add .S (preprocessed assembly) to C compiler source extensions. - self.compiler.src_extensions.append('.S') - - include_dirs = [os.path.join(ffi_srcdir, 'include'), - os.path.join(ffi_srcdir, 'powerpc')] - ext.include_dirs.extend(include_dirs) - ext.sources.extend(sources) - return True - def configure_ctypes(self, ext): - if not self.use_system_libffi: - if MACOS: - return self.configure_ctypes_darwin(ext) - print('INFO: Could not locate ffi libs and/or headers') - return False return True def detect_ctypes(self): # Thomas Heller's _ctypes module - self.use_system_libffi = False + + if (not sysconfig.get_config_var("LIBFFI_INCLUDEDIR") and MACOS): + self.use_system_libffi = True + else: + self.use_system_libffi = '--with-system-ffi' in sysconfig.get_config_var("CONFIG_ARGS") + include_dirs = [] extra_compile_args = ['-DPy_BUILD_CORE_MODULE'] extra_link_args = [] @@ -2154,11 +2135,9 @@ class PyBuildExt(build_ext): if MACOS: sources.append('_ctypes/malloc_closure.c') - sources.append('_ctypes/darwin/dlfcn_simple.c') + extra_compile_args.append('-DUSING_MALLOC_CLOSURE_DOT_C=1') extra_compile_args.append('-DMACOSX') include_dirs.append('_ctypes/darwin') - # XXX Is this still needed? - # extra_link_args.extend(['-read_only_relocs', 'warning']) elif HOST_PLATFORM == 'sunos5': # XXX This shouldn't be necessary; it appears that some @@ -2188,31 +2167,48 @@ class PyBuildExt(build_ext): sources=['_ctypes/_ctypes_test.c'], libraries=['m'])) + ffi_inc = sysconfig.get_config_var("LIBFFI_INCLUDEDIR") + ffi_lib = None + ffi_inc_dirs = self.inc_dirs.copy() if MACOS: - if '--with-system-ffi' not in sysconfig.get_config_var("CONFIG_ARGS"): - return - # OS X 10.5 comes with libffi.dylib; the include files are - # in /usr/include/ffi - ffi_inc_dirs.append('/usr/include/ffi') + ffi_in_sdk = os.path.join(macosx_sdk_root(), "usr/include/ffi") - ffi_inc = [sysconfig.get_config_var("LIBFFI_INCLUDEDIR")] - if not ffi_inc or ffi_inc[0] == '': - ffi_inc = find_file('ffi.h', [], ffi_inc_dirs) - if ffi_inc is not None: - ffi_h = ffi_inc[0] + '/ffi.h' + if not ffi_inc: + if os.path.exists(ffi_in_sdk): + ext.extra_compile_args.append("-DUSING_APPLE_OS_LIBFFI=1") + ffi_inc = ffi_in_sdk + ffi_lib = 'ffi' + else: + # OS X 10.5 comes with libffi.dylib; the include files are + # in /usr/include/ffi + ffi_inc_dirs.append('/usr/include/ffi') + + if not ffi_inc: + found = find_file('ffi.h', [], ffi_inc_dirs) + if found: + ffi_inc = found[0] + if ffi_inc: + ffi_h = ffi_inc + '/ffi.h' if not os.path.exists(ffi_h): ffi_inc = None print('Header file {} does not exist'.format(ffi_h)) - ffi_lib = None - if ffi_inc is not None: + if ffi_lib is None and ffi_inc: for lib_name in ('ffi', 'ffi_pic'): if (self.compiler.find_library_file(self.lib_dirs, lib_name)): ffi_lib = lib_name break if ffi_inc and ffi_lib: - ext.include_dirs.extend(ffi_inc) + ffi_headers = glob(os.path.join(ffi_inc, '*.h')) + if grep_headers_for('ffi_prep_cif_var', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CIF_VAR=1") + if grep_headers_for('ffi_prep_closure_loc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_PREP_CLOSURE_LOC=1") + if grep_headers_for('ffi_closure_alloc', ffi_headers): + ext.extra_compile_args.append("-DHAVE_FFI_CLOSURE_ALLOC=1") + + ext.include_dirs.append(ffi_inc) ext.libraries.append(ffi_lib) self.use_system_libffi = True From 23831a7a90956e38b7d70304bb6afe30d37936de Mon Sep 17 00:00:00 2001 From: Ronald Oussoren Date: Sun, 8 Nov 2020 10:46:55 +0100 Subject: [PATCH 48/61] bpo-41754: Ignore NotADirectoryError in invocation of xdg-settings (GH-23075) It is not clear why this can happen, but several users have mentioned getting this exception on macOS. --- Lib/webbrowser.py | 2 +- .../next/Library/2020-11-01-15-07-20.bpo-41754.DraSZh.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-01-15-07-20.bpo-41754.DraSZh.rst diff --git a/Lib/webbrowser.py b/Lib/webbrowser.py index cea91308ce1..6023c1e1384 100755 --- a/Lib/webbrowser.py +++ b/Lib/webbrowser.py @@ -550,7 +550,7 @@ def register_standard_browsers(): cmd = "xdg-settings get default-web-browser".split() raw_result = subprocess.check_output(cmd, stderr=subprocess.DEVNULL) result = raw_result.decode().strip() - except (FileNotFoundError, subprocess.CalledProcessError, PermissionError) : + except (FileNotFoundError, subprocess.CalledProcessError, PermissionError, NotADirectoryError) : pass else: global _os_preferred_browser diff --git a/Misc/NEWS.d/next/Library/2020-11-01-15-07-20.bpo-41754.DraSZh.rst b/Misc/NEWS.d/next/Library/2020-11-01-15-07-20.bpo-41754.DraSZh.rst new file mode 100644 index 00000000000..181c2d9650a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-01-15-07-20.bpo-41754.DraSZh.rst @@ -0,0 +1 @@ +webbrowser: Ignore *NotADirectoryError* when calling ``xdg-settings``. From 4eb41d055e8307b8206f680287e492a6db068acd Mon Sep 17 00:00:00 2001 From: kj <28750310+Fidget-Spinner@users.noreply.github.com> Date: Mon, 9 Nov 2020 12:00:13 +0800 Subject: [PATCH 49/61] bpo-42233: Add union type expression support for GenericAlias and fix de-duplicating of GenericAlias (GH-23077) --- Include/internal/pycore_unionobject.h | 1 + Lib/test/test_types.py | 22 +++++++++++++++++++ .../2020-11-01-23-34-56.bpo-42233.zOSzja.rst | 5 +++++ Objects/genericaliasobject.c | 6 +++++ Objects/typeobject.c | 14 ++---------- Objects/unionobject.c | 20 ++++++++++++----- 6 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2020-11-01-23-34-56.bpo-42233.zOSzja.rst diff --git a/Include/internal/pycore_unionobject.h b/Include/internal/pycore_unionobject.h index fa8ba6ed944..4d82b6fbeae 100644 --- a/Include/internal/pycore_unionobject.h +++ b/Include/internal/pycore_unionobject.h @@ -10,6 +10,7 @@ extern "C" { PyAPI_FUNC(PyObject *) _Py_Union(PyObject *args); PyAPI_DATA(PyTypeObject) _Py_UnionType; +PyAPI_FUNC(PyObject *) _Py_union_type_or(PyObject* self, PyObject* param); #ifdef __cplusplus } diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 75c5eee42dc..3058a02d6ee 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -713,6 +713,28 @@ class TypesTests(unittest.TestCase): assert repr(int | None) == "int | None" assert repr(int | typing.GenericAlias(list, int)) == "int | list[int]" + def test_or_type_operator_with_genericalias(self): + a = list[int] + b = list[str] + c = dict[float, str] + # equivalence with typing.Union + self.assertEqual(a | b | c, typing.Union[a, b, c]) + # de-duplicate + self.assertEqual(a | c | b | b | a | c, a | b | c) + # order shouldn't matter + self.assertEqual(a | b, b | a) + self.assertEqual(repr(a | b | c), + "list[int] | list[str] | dict[float, str]") + + class BadType(type): + def __eq__(self, other): + return 1 / 0 + + bt = BadType('bt', (), {}) + # Comparison should fail and errors should propagate out for bad types. + with self.assertRaises(ZeroDivisionError): + list[int] | list[bt] + def test_ellipsis_type(self): self.assertIsInstance(Ellipsis, types.EllipsisType) diff --git a/Misc/NEWS.d/next/Core and Builtins/2020-11-01-23-34-56.bpo-42233.zOSzja.rst b/Misc/NEWS.d/next/Core and Builtins/2020-11-01-23-34-56.bpo-42233.zOSzja.rst new file mode 100644 index 00000000000..499bb324fb9 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2020-11-01-23-34-56.bpo-42233.zOSzja.rst @@ -0,0 +1,5 @@ +Allow ``GenericAlias`` objects to use :ref:`union type expressions `. +This allows expressions like ``list[int] | dict[float, str]`` where previously a +``TypeError`` would have been thrown. This also fixes union type expressions +not de-duplicating ``GenericAlias`` objects. (Contributed by Ken Jin in +:issue:`42233`.) diff --git a/Objects/genericaliasobject.c b/Objects/genericaliasobject.c index 6508c69cbf7..28ea487a44f 100644 --- a/Objects/genericaliasobject.c +++ b/Objects/genericaliasobject.c @@ -2,6 +2,7 @@ #include "Python.h" #include "pycore_object.h" +#include "pycore_unionobject.h" // _Py_union_as_number #include "structmember.h" // PyMemberDef typedef struct { @@ -573,6 +574,10 @@ ga_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return Py_GenericAlias(origin, arguments); } +static PyNumberMethods ga_as_number = { + .nb_or = (binaryfunc)_Py_union_type_or, // Add __or__ function +}; + // TODO: // - argument clinic? // - __doc__? @@ -586,6 +591,7 @@ PyTypeObject Py_GenericAliasType = { .tp_basicsize = sizeof(gaobject), .tp_dealloc = ga_dealloc, .tp_repr = ga_repr, + .tp_as_number = &ga_as_number, // allow X | Y of GenericAlias objs .tp_as_mapping = &ga_as_mapping, .tp_hash = ga_hash, .tp_call = ga_call, diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 3822b8cf813..55bf9b3f389 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -6,7 +6,7 @@ #include "pycore_object.h" #include "pycore_pyerrors.h" #include "pycore_pystate.h" // _PyThreadState_GET() -#include "pycore_unionobject.h" // _Py_Union() +#include "pycore_unionobject.h" // _Py_Union(), _Py_union_type_or #include "frameobject.h" #include "structmember.h" // PyMemberDef @@ -3789,19 +3789,9 @@ type_is_gc(PyTypeObject *type) return type->tp_flags & Py_TPFLAGS_HEAPTYPE; } -static PyObject * -type_or(PyTypeObject* self, PyObject* param) { - PyObject *tuple = PyTuple_Pack(2, self, param); - if (tuple == NULL) { - return NULL; - } - PyObject *new_union = _Py_Union(tuple); - Py_DECREF(tuple); - return new_union; -} static PyNumberMethods type_as_number = { - .nb_or = (binaryfunc)type_or, // Add __or__ function + .nb_or = _Py_union_type_or, // Add __or__ function }; PyTypeObject PyType_Type = { diff --git a/Objects/unionobject.c b/Objects/unionobject.c index 1b7f8ab51a4..2308bfc9f2a 100644 --- a/Objects/unionobject.c +++ b/Objects/unionobject.c @@ -237,9 +237,19 @@ dedup_and_flatten_args(PyObject* args) PyObject* i_element = PyTuple_GET_ITEM(args, i); for (Py_ssize_t j = i + 1; j < arg_length; j++) { PyObject* j_element = PyTuple_GET_ITEM(args, j); - if (i_element == j_element) { - is_duplicate = 1; + int is_ga = Py_TYPE(i_element) == &Py_GenericAliasType && + Py_TYPE(j_element) == &Py_GenericAliasType; + // RichCompare to also deduplicate GenericAlias types (slower) + is_duplicate = is_ga ? PyObject_RichCompareBool(i_element, j_element, Py_EQ) + : i_element == j_element; + // Should only happen if RichCompare fails + if (is_duplicate < 0) { + Py_DECREF(args); + Py_DECREF(new_args); + return NULL; } + if (is_duplicate) + break; } if (!is_duplicate) { Py_INCREF(i_element); @@ -290,8 +300,8 @@ is_unionable(PyObject *obj) type == &_Py_UnionType); } -static PyObject * -type_or(PyTypeObject* self, PyObject* param) +PyObject * +_Py_union_type_or(PyObject* self, PyObject* param) { PyObject *tuple = PyTuple_Pack(2, self, param); if (tuple == NULL) { @@ -404,7 +414,7 @@ static PyMethodDef union_methods[] = { {0}}; static PyNumberMethods union_as_number = { - .nb_or = (binaryfunc)type_or, // Add __or__ function + .nb_or = _Py_union_type_or, // Add __or__ function }; PyTypeObject _Py_UnionType = { From 97e8b1eaeaf3aa325c84ff2e13417c30414d0269 Mon Sep 17 00:00:00 2001 From: Ammar Askar Date: Mon, 9 Nov 2020 02:02:39 -0500 Subject: [PATCH 50/61] bpo-40624: Add support for the XPath != operator in xml.etree (GH-22147) --- Doc/library/xml.etree.elementtree.rst | 18 ++++++++++ Lib/test/test_xml_etree.py | 35 +++++++++++++++++++ Lib/xml/etree/ElementPath.py | 31 ++++++++++++---- .../2020-09-08-03-19-04.bpo-40624.0-gYfx.rst | 1 + 4 files changed, 78 insertions(+), 7 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-09-08-03-19-04.bpo-40624.0-gYfx.rst diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst index f4bccf66098..87f4ee347d6 100644 --- a/Doc/library/xml.etree.elementtree.rst +++ b/Doc/library/xml.etree.elementtree.rst @@ -455,6 +455,12 @@ Supported XPath syntax | | has the given value. The value cannot contain | | | quotes. | +-----------------------+------------------------------------------------------+ +| ``[@attrib!='value']``| Selects all elements for which the given attribute | +| | does not have the given value. The value cannot | +| | contain quotes. | +| | | +| | .. versionadded:: 3.10 | ++-----------------------+------------------------------------------------------+ | ``[tag]`` | Selects all elements that have a child named | | | ``tag``. Only immediate children are supported. | +-----------------------+------------------------------------------------------+ @@ -463,10 +469,22 @@ Supported XPath syntax | | | | | .. versionadded:: 3.7 | +-----------------------+------------------------------------------------------+ +| ``[.!='text']`` | Selects all elements whose complete text content, | +| | including descendants, does not equal the given | +| | ``text``. | +| | | +| | .. versionadded:: 3.10 | ++-----------------------+------------------------------------------------------+ | ``[tag='text']`` | Selects all elements that have a child named | | | ``tag`` whose complete text content, including | | | descendants, equals the given ``text``. | +-----------------------+------------------------------------------------------+ +| ``[tag!='text']`` | Selects all elements that have a child named | +| | ``tag`` whose complete text content, including | +| | descendants, does not equal the given ``text``. | +| | | +| | .. versionadded:: 3.10 | ++-----------------------+------------------------------------------------------+ | ``[position]`` | Selects all elements that are located at the given | | | position. The position can be either an integer | | | (1 is the first position), the expression ``last()`` | diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 3f1f3781e48..fd4a38527fd 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -2852,8 +2852,12 @@ class ElementFindTest(unittest.TestCase): ['tag'] * 3) self.assertEqual(summarize_list(e.findall('.//tag[@class="a"]')), ['tag']) + self.assertEqual(summarize_list(e.findall('.//tag[@class!="a"]')), + ['tag'] * 2) self.assertEqual(summarize_list(e.findall('.//tag[@class="b"]')), ['tag'] * 2) + self.assertEqual(summarize_list(e.findall('.//tag[@class!="b"]')), + ['tag']) self.assertEqual(summarize_list(e.findall('.//tag[@id]')), ['tag']) self.assertEqual(summarize_list(e.findall('.//section[tag]')), @@ -2875,6 +2879,19 @@ class ElementFindTest(unittest.TestCase): self.assertEqual(summarize_list(e.findall(".//section[ tag = 'subtext' ]")), ['section']) + # Negations of above tests. They match nothing because the sole section + # tag has subtext. + self.assertEqual(summarize_list(e.findall(".//section[tag!='subtext']")), + []) + self.assertEqual(summarize_list(e.findall(".//section[tag !='subtext']")), + []) + self.assertEqual(summarize_list(e.findall(".//section[tag!= 'subtext']")), + []) + self.assertEqual(summarize_list(e.findall(".//section[tag != 'subtext']")), + []) + self.assertEqual(summarize_list(e.findall(".//section[ tag != 'subtext' ]")), + []) + self.assertEqual(summarize_list(e.findall(".//tag[.='subtext']")), ['tag']) self.assertEqual(summarize_list(e.findall(".//tag[. ='subtext']")), @@ -2890,6 +2907,24 @@ class ElementFindTest(unittest.TestCase): self.assertEqual(summarize_list(e.findall(".//tag[.= ' subtext']")), []) + # Negations of above tests. + # Matches everything but the tag containing subtext + self.assertEqual(summarize_list(e.findall(".//tag[.!='subtext']")), + ['tag'] * 3) + self.assertEqual(summarize_list(e.findall(".//tag[. !='subtext']")), + ['tag'] * 3) + self.assertEqual(summarize_list(e.findall('.//tag[.!= "subtext"]')), + ['tag'] * 3) + self.assertEqual(summarize_list(e.findall('.//tag[ . != "subtext" ]')), + ['tag'] * 3) + self.assertEqual(summarize_list(e.findall(".//tag[. != 'subtext']")), + ['tag'] * 3) + # Matches all tags. + self.assertEqual(summarize_list(e.findall(".//tag[. != 'subtext ']")), + ['tag'] * 4) + self.assertEqual(summarize_list(e.findall(".//tag[.!= ' subtext']")), + ['tag'] * 4) + # duplicate section => 2x tag matches e[1] = e[2] self.assertEqual(summarize_list(e.findall(".//section[tag = 'subtext']")), diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index d318e65d84a..1cbd8399d14 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -65,8 +65,9 @@ xpath_tokenizer_re = re.compile( r"//?|" r"\.\.|" r"\(\)|" + r"!=|" r"[/.*:\[\]\(\)@=])|" - r"((?:\{[^}]+\})?[^/\[\]\(\)@=\s]+)|" + r"((?:\{[^}]+\})?[^/\[\]\(\)@!=\s]+)|" r"\s+" ) @@ -253,15 +254,19 @@ def prepare_predicate(next, token): if elem.get(key) is not None: yield elem return select - if signature == "@-='": - # [@attribute='value'] + if signature == "@-='" or signature == "@-!='": + # [@attribute='value'] or [@attribute!='value'] key = predicate[1] value = predicate[-1] def select(context, result): for elem in result: if elem.get(key) == value: yield elem - return select + def select_negated(context, result): + for elem in result: + if (attr_value := elem.get(key)) is not None and attr_value != value: + yield elem + return select_negated if '!=' in signature else select if signature == "-" and not re.match(r"\-?\d+$", predicate[0]): # [tag] tag = predicate[0] @@ -270,8 +275,10 @@ def prepare_predicate(next, token): if elem.find(tag) is not None: yield elem return select - if signature == ".='" or (signature == "-='" and not re.match(r"\-?\d+$", predicate[0])): - # [.='value'] or [tag='value'] + if signature == ".='" or signature == ".!='" or ( + (signature == "-='" or signature == "-!='") + and not re.match(r"\-?\d+$", predicate[0])): + # [.='value'] or [tag='value'] or [.!='value'] or [tag!='value'] tag = predicate[0] value = predicate[-1] if tag: @@ -281,12 +288,22 @@ def prepare_predicate(next, token): if "".join(e.itertext()) == value: yield elem break + def select_negated(context, result): + for elem in result: + for e in elem.iterfind(tag): + if "".join(e.itertext()) != value: + yield elem + break else: def select(context, result): for elem in result: if "".join(elem.itertext()) == value: yield elem - return select + def select_negated(context, result): + for elem in result: + if "".join(elem.itertext()) != value: + yield elem + return select_negated if '!=' in signature else select if signature == "-" or signature == "-()" or signature == "-()-": # [index] or [last()] or [last()-index] if signature == "-": diff --git a/Misc/NEWS.d/next/Library/2020-09-08-03-19-04.bpo-40624.0-gYfx.rst b/Misc/NEWS.d/next/Library/2020-09-08-03-19-04.bpo-40624.0-gYfx.rst new file mode 100644 index 00000000000..78bad6e4686 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-09-08-03-19-04.bpo-40624.0-gYfx.rst @@ -0,0 +1 @@ +Added support for the XPath ``!=`` operator in xml.etree From a117167d8dc8fa673a4646f509551c7950f824e5 Mon Sep 17 00:00:00 2001 From: Tom Gringauz Date: Mon, 9 Nov 2020 14:34:07 +0200 Subject: [PATCH 51/61] bpo-41543: contextlib.nullcontext can fill in for an async context manager (GH-21870) Co-authored-by: Andrew Svetlov --- Doc/library/contextlib.rst | 18 ++++++++++++++++++ Lib/contextlib.py | 8 +++++++- Lib/test/test_contextlib_async.py | 14 +++++++++++++- .../2020-08-14-00-39-04.bpo-41543.RpcRjb.rst | 1 + 4 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-08-14-00-39-04.bpo-41543.RpcRjb.rst diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst index d5a1068a734..91edbba7283 100644 --- a/Doc/library/contextlib.rst +++ b/Doc/library/contextlib.rst @@ -243,8 +243,26 @@ Functions and classes provided: with cm as file: # Perform processing on the file + It can also be used as a stand-in for + :ref:`asynchronous context managers `:: + + async def send_http(session=None): + if not session: + # If no http session, create it with aiohttp + cm = aiohttp.ClientSession() + else: + # Caller is responsible for closing the session + cm = nullcontext(session) + + async with cm as session: + # Send http requests with session + .. versionadded:: 3.7 + .. versionchanged:: 3.10 + :term:`asynchronous context manager` support was added. + + .. function:: suppress(*exceptions) diff --git a/Lib/contextlib.py b/Lib/contextlib.py index 56b4968118b..a0b523c96fb 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -704,7 +704,7 @@ class AsyncExitStack(_BaseExitStack, AbstractAsyncContextManager): return received_exc and suppressed_exc -class nullcontext(AbstractContextManager): +class nullcontext(AbstractContextManager, AbstractAsyncContextManager): """Context manager that does no additional processing. Used as a stand-in for a normal context manager, when a particular @@ -723,3 +723,9 @@ class nullcontext(AbstractContextManager): def __exit__(self, *excinfo): pass + + async def __aenter__(self): + return self.enter_result + + async def __aexit__(self, *excinfo): + pass diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py index 109807d633d..290ef05b82a 100644 --- a/Lib/test/test_contextlib_async.py +++ b/Lib/test/test_contextlib_async.py @@ -1,5 +1,7 @@ import asyncio -from contextlib import aclosing, asynccontextmanager, AbstractAsyncContextManager, AsyncExitStack +from contextlib import ( + asynccontextmanager, AbstractAsyncContextManager, + AsyncExitStack, nullcontext, aclosing) import functools from test import support import unittest @@ -537,5 +539,15 @@ class TestAsyncExitStack(TestBaseExitStack, unittest.TestCase): self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) +class TestAsyncNullcontext(unittest.TestCase): + @_async_test + async def test_async_nullcontext(self): + class C: + pass + c = C() + async with nullcontext(c) as c_in: + self.assertIs(c_in, c) + + if __name__ == '__main__': unittest.main() diff --git a/Misc/NEWS.d/next/Library/2020-08-14-00-39-04.bpo-41543.RpcRjb.rst b/Misc/NEWS.d/next/Library/2020-08-14-00-39-04.bpo-41543.RpcRjb.rst new file mode 100644 index 00000000000..753dc763f21 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-08-14-00-39-04.bpo-41543.RpcRjb.rst @@ -0,0 +1 @@ +Add async context manager support for contextlib.nullcontext. From 23c5f93b83f78f295313e137011edb18b24c37c2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 9 Nov 2020 13:40:47 +0100 Subject: [PATCH 52/61] bpo-42294: Add borrowed/strong reference to doc glossary (GH-23206) Add "borrowed reference" and "strong reference" to the documentation glossary. Enhance also Py_INCREF() and Py_NewRef() documentation. --- Doc/c-api/arg.rst | 3 ++- Doc/c-api/init.rst | 2 +- Doc/c-api/intro.rst | 2 +- Doc/c-api/refcounting.rst | 35 ++++++++++++++++++++++++++--------- Doc/c-api/reflection.rst | 4 ++-- Doc/c-api/structures.rst | 2 +- Doc/c-api/typeobj.rst | 5 +++-- Doc/c-api/weakref.rst | 6 +++--- Doc/data/refcounts.dat | 6 ++++++ Doc/glossary.rst | 24 ++++++++++++++++++++++++ Include/object.h | 5 +++-- 11 files changed, 72 insertions(+), 22 deletions(-) diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst index bdaae44e240..a91b3c7c950 100644 --- a/Doc/c-api/arg.rst +++ b/Doc/c-api/arg.rst @@ -482,7 +482,8 @@ API Functions *min* and no more than *max*; *min* and *max* may be equal. Additional arguments must be passed to the function, each of which should be a pointer to a :c:type:`PyObject*` variable; these will be filled in with the values from - *args*; they will contain borrowed references. The variables which correspond + *args*; they will contain :term:`borrowed references `. + The variables which correspond to optional parameters not given by *args* will not be filled in; these should be initialized by the caller. This function returns true on success and false if *args* is not a tuple or contains the wrong number of elements; an exception diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst index 3ce689203a8..3d18bb3f0b9 100644 --- a/Doc/c-api/init.rst +++ b/Doc/c-api/init.rst @@ -1077,7 +1077,7 @@ All of the following functions must be called after :c:func:`Py_Initialize`. Get the current frame of the Python thread state *tstate*. - Return a strong reference. Return ``NULL`` if no frame is currently + Return a :term:`strong reference`. Return ``NULL`` if no frame is currently executing. See also :c:func:`PyEval_GetFrame`. diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst index 7ca8693afab..bae5ce11b73 100644 --- a/Doc/c-api/intro.rst +++ b/Doc/c-api/intro.rst @@ -326,7 +326,7 @@ when it's no longer needed---or passing on this responsibility (usually to its caller). When a function passes ownership of a reference on to its caller, the caller is said to receive a *new* reference. When no ownership is transferred, the caller is said to *borrow* the reference. Nothing needs to be done for a -borrowed reference. +:term:`borrowed reference`. Conversely, when a calling function passes in a reference to an object, there are two possibilities: the function *steals* a reference to the object, or it diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst index b15c0e6aecc..391907c8c29 100644 --- a/Doc/c-api/refcounting.rst +++ b/Doc/c-api/refcounting.rst @@ -13,10 +13,14 @@ objects. .. c:function:: void Py_INCREF(PyObject *o) - Increment the reference count for object *o*. The object must not be ``NULL``; if - you aren't sure that it isn't ``NULL``, use :c:func:`Py_XINCREF`. + Increment the reference count for object *o*. - See also :c:func:`Py_NewRef`. + This function is usually used to convert a :term:`borrowed reference` to a + :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be + used to create a new :term:`strong reference`. + + The object must not be ``NULL``; if you aren't sure that it isn't + ``NULL``, use :c:func:`Py_XINCREF`. .. c:function:: void Py_XINCREF(PyObject *o) @@ -29,9 +33,14 @@ objects. .. c:function:: PyObject* Py_NewRef(PyObject *o) - Increment the reference count of the object *o* and return the object *o*. + Create a new :term:`strong reference` to an object: increment the reference + count of the object *o* and return the object *o*. - The object *o* must not be ``NULL``. + When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` + should be called on it to decrement the object reference count. + + The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be + ``NULL``. For example:: @@ -42,6 +51,8 @@ objects. self->attr = Py_NewRef(obj); + See also :c:func:`Py_INCREF`. + .. versionadded:: 3.10 @@ -56,10 +67,16 @@ objects. .. c:function:: void Py_DECREF(PyObject *o) - Decrement the reference count for object *o*. The object must not be ``NULL``; if - you aren't sure that it isn't ``NULL``, use :c:func:`Py_XDECREF`. If the reference - count reaches zero, the object's type's deallocation function (which must not be - ``NULL``) is invoked. + Decrement the reference count for object *o*. + + If the reference count reaches zero, the object's type's deallocation + function (which must not be ``NULL``) is invoked. + + This function is usually used to delete a :term:`strong reference` before + exiting its scope. + + The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, + use :c:func:`Py_XDECREF`. .. warning:: diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst index 9207d86012c..64ce4d1d0c3 100644 --- a/Doc/c-api/reflection.rst +++ b/Doc/c-api/reflection.rst @@ -35,7 +35,7 @@ Reflection Get the *frame* next outer frame. - Return a strong reference, or ``NULL`` if *frame* has no outer frame. + Return a :term:`strong reference`, or ``NULL`` if *frame* has no outer frame. *frame* must not be ``NULL``. @@ -46,7 +46,7 @@ Reflection Get the *frame* code. - Return a strong reference. + Return a :term:`strong reference`. *frame* must not be ``NULL``. The result (frame code) cannot be ``NULL``. diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index a9e1c6fbcc3..03fe479165f 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -66,7 +66,7 @@ the definition of all other Python objects. Get the type of the Python object *o*. - Return a borrowed reference. + Return a :term:`borrowed reference`. .. versionchanged:: 3.10 :c:func:`Py_TYPE()` is changed to the inline static function. diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst index ddcb8ae3d09..32bbc7ba0a1 100644 --- a/Doc/c-api/typeobj.rst +++ b/Doc/c-api/typeobj.rst @@ -1213,8 +1213,9 @@ and :c:type:`PyType_Type` effectively act as defaults.) :func:`~gc.get_referents` function will include it. .. warning:: - When implementing :c:member:`~PyTypeObject.tp_traverse`, only the members - that the instance *owns* (by having strong references to them) must be + When implementing :c:member:`~PyTypeObject.tp_traverse`, only the + members that the instance *owns* (by having :term:`strong references + ` to them) must be visited. For instance, if an object supports weak references via the :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting the linked list (what *tp_weaklist* points to) must **not** be diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst index e3a9bda54d6..fb6628a1bbf 100644 --- a/Doc/c-api/weakref.rst +++ b/Doc/c-api/weakref.rst @@ -57,10 +57,10 @@ as much as it can. .. note:: - This function returns a **borrowed reference** to the referenced object. + This function returns a :term:`borrowed reference` to the referenced object. This means that you should always call :c:func:`Py_INCREF` on the object - except if you know that it cannot be destroyed while you are still - using it. + except it cannot be destroyed before the last usage of the borrowed + reference. .. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat index d01e99ca5e3..8a6ee718a01 100644 --- a/Doc/data/refcounts.dat +++ b/Doc/data/refcounts.dat @@ -3007,6 +3007,9 @@ Py_GetVersion:const char*::: Py_INCREF:void::: Py_INCREF:PyObject*:o:+1: +Py_NewRef:void::: +Py_NewRef:PyObject*:o:+1: + Py_Initialize:void::: Py_IsInitialized:int::: @@ -3028,6 +3031,9 @@ Py_XDECREF:PyObject*:o:-1:if o is not NULL Py_XINCREF:void::: Py_XINCREF:PyObject*:o:+1:if o is not NULL +Py_XNewRef:void::: +Py_XNewRef:PyObject*:o:+1:if o is not NULL + _PyImport_Fini:void::: _PyObject_New:PyObject*::+1: diff --git a/Doc/glossary.rst b/Doc/glossary.rst index 506973e964b..b410585ca81 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -158,6 +158,18 @@ Glossary See also :term:`text file` for a file object able to read and write :class:`str` objects. + borrowed reference + In the Python's C API, a borrowed reference is a reference to an object. + It does not modify the object reference count. It becomes a dangling + pointer if the object is destroyed. For example, a garbage collection can + remove the last :term:`strong reference` to the object and so destroy it. + + Calling :c:func:`Py_INCREF` on the :term:`borrowed reference` is + recommended to convert it to a :term:`strong reference` in-place, except + if the object cannot be destroyed before the last usage of the borrowed + reference. The :c:func:`Py_NewRef` function can be used to create a new + :term:`strong reference`. + bytes-like object An object that supports the :ref:`bufferobjects` and can export a C-:term:`contiguous` buffer. This includes all :class:`bytes`, @@ -1100,6 +1112,18 @@ Glossary an :term:`expression` or one of several constructs with a keyword, such as :keyword:`if`, :keyword:`while` or :keyword:`for`. + strong reference + In the Python's C API, a strong reference is a reference to an object + which increments object reference count when it is created and + decrements the object reference count when it is deleted. + + The :c:func:`Py_NewRef` function can be used to create a strong reference + to an object. Usually, the :c:func:`Py_DECREF` function must be called on + the strong reference before exiting the scope of the strong reference, to + avoid leaking one reference. + + See also :term:`borrowed reference`. + text encoding A codec which encodes Unicode strings to bytes. diff --git a/Include/object.h b/Include/object.h index 835d9de01fb..eab3228f3ab 100644 --- a/Include/object.h +++ b/Include/object.h @@ -526,10 +526,11 @@ they can have object code that is not dependent on Python compilation flags. PyAPI_FUNC(void) Py_IncRef(PyObject *); PyAPI_FUNC(void) Py_DecRef(PyObject *); -// Increment the reference count of the object and return the object. +// Create a new strong reference to an object: +// increment the reference count of the object and return the object. PyAPI_FUNC(PyObject*) Py_NewRef(PyObject *obj); -// Similar to Py_NewRef() but the object can be NULL. +// Similar to Py_NewRef(), but the object can be NULL. PyAPI_FUNC(PyObject*) Py_XNewRef(PyObject *obj); static inline PyObject* _Py_NewRef(PyObject *obj) From 1f73c320e2921605c4963e202f6bdac1ef18f2ce Mon Sep 17 00:00:00 2001 From: Yash Shete Date: Mon, 9 Nov 2020 23:08:09 +0530 Subject: [PATCH 53/61] bpo-41712: Avoid runaway regex match in upload scripts (GH-23166) --- Tools/msi/purge.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/msi/purge.py b/Tools/msi/purge.py index a8b8f4d8973..27b6b054a44 100644 --- a/Tools/msi/purge.py +++ b/Tools/msi/purge.py @@ -12,7 +12,7 @@ import sys from urllib.request import * -VERSION_RE = re.compile(r'(\d+\.\d+\.\d+)(\w+\d+)?$') +VERSION_RE = re.compile(r'(\d+\.\d+\.\d+)([A-Za-z_]+\d+)?$') try: m = VERSION_RE.match(sys.argv[1]) From bfc6b63102d37ccb58a71711e2342143cd9f4d86 Mon Sep 17 00:00:00 2001 From: jack1142 <6032823+jack1142@users.noreply.github.com> Date: Mon, 9 Nov 2020 23:50:45 +0100 Subject: [PATCH 54/61] bpo-36310: Allow pygettext.py to detect calls to gettext in f-strings. (GH-19875) Adds support to Tools/i18n/pygettext.py for gettext calls in f-strings. This process is done by parsing the f-strings, processing each value, and flagging the ones which contain a gettext call. Co-authored-by: Batuhan Taskaya --- Lib/test/test_tools/test_i18n.py | 70 +++++++++++++++++++ Misc/ACKS | 1 + .../2020-05-03-01-30-46.bpo-36310.xDxxwY.rst | 2 + Tools/i18n/pygettext.py | 53 ++++++++++++++ 4 files changed, 126 insertions(+) create mode 100644 Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst diff --git a/Lib/test/test_tools/test_i18n.py b/Lib/test/test_tools/test_i18n.py index 8da657907ea..12f778dbf84 100644 --- a/Lib/test/test_tools/test_i18n.py +++ b/Lib/test/test_tools/test_i18n.py @@ -220,6 +220,76 @@ class Test_pygettext(unittest.TestCase): ''')) self.assertIn('doc', msgids) + def test_calls_in_fstrings(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_raw(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + rf"{_('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_nested(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"""{f'{_("foo bar")}'}""" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_attribute(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{obj._('foo bar')}" + ''')) + self.assertIn('foo bar', msgids) + + def test_calls_in_fstrings_with_call_on_call(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{type(str)('foo bar')}" + ''')) + self.assertNotIn('foo bar', msgids) + + def test_calls_in_fstrings_with_format(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo {bar}').format(bar='baz')}" + ''')) + self.assertIn('foo {bar}', msgids) + + def test_calls_in_fstrings_with_wrong_input_1(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(f'foo {bar}')}" + ''')) + self.assertFalse([msgid for msgid in msgids if 'foo {bar}' in msgid]) + + def test_calls_in_fstrings_with_wrong_input_2(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(1)}" + ''')) + self.assertNotIn(1, msgids) + + def test_calls_in_fstring_with_multiple_args(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo', 'bar')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertNotIn('bar', msgids) + + def test_calls_in_fstring_with_keyword_args(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_('foo', bar='baz')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertNotIn('bar', msgids) + self.assertNotIn('baz', msgids) + + def test_calls_in_fstring_with_partially_wrong_expression(self): + msgids = self.extract_docstrings_from_str(dedent('''\ + f"{_(f'foo') + _('bar')}" + ''')) + self.assertNotIn('foo', msgids) + self.assertIn('bar', msgids) + def test_files_list(self): """Make sure the directories are inspected for source files bpo-31920 diff --git a/Misc/ACKS b/Misc/ACKS index 5285693a6e6..35a87ae6b96 100644 --- a/Misc/ACKS +++ b/Misc/ACKS @@ -949,6 +949,7 @@ Ivan Krstić Anselm Kruis Steven Kryskalla Andrew Kuchling +Jakub Kuczys Dave Kuhlman Jon Kuhn Ilya Kulakov diff --git a/Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst b/Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst new file mode 100644 index 00000000000..16749a8fc96 --- /dev/null +++ b/Misc/NEWS.d/next/Tools-Demos/2020-05-03-01-30-46.bpo-36310.xDxxwY.rst @@ -0,0 +1,2 @@ +Allow :file:`Tools/i18n/pygettext.py` to detect calls to ``gettext`` in +f-strings. \ No newline at end of file diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index b1d281d793b..9dd65c221fd 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -162,6 +162,7 @@ import sys import glob import time import getopt +import ast import token import tokenize @@ -343,6 +344,58 @@ class TokenEater: return if ttype == tokenize.NAME and tstring in opts.keywords: self.__state = self.__keywordseen + return + if ttype == tokenize.STRING: + maybe_fstring = ast.parse(tstring, mode='eval').body + if not isinstance(maybe_fstring, ast.JoinedStr): + return + for value in filter(lambda node: isinstance(node, ast.FormattedValue), + maybe_fstring.values): + for call in filter(lambda node: isinstance(node, ast.Call), + ast.walk(value)): + func = call.func + if isinstance(func, ast.Name): + func_name = func.id + elif isinstance(func, ast.Attribute): + func_name = func.attr + else: + continue + + if func_name not in opts.keywords: + continue + if len(call.args) != 1: + print(_( + '*** %(file)s:%(lineno)s: Seen unexpected amount of' + ' positional arguments in gettext call: %(source_segment)s' + ) % { + 'source_segment': ast.get_source_segment(tstring, call) or tstring, + 'file': self.__curfile, + 'lineno': lineno + }, file=sys.stderr) + continue + if call.keywords: + print(_( + '*** %(file)s:%(lineno)s: Seen unexpected keyword arguments' + ' in gettext call: %(source_segment)s' + ) % { + 'source_segment': ast.get_source_segment(tstring, call) or tstring, + 'file': self.__curfile, + 'lineno': lineno + }, file=sys.stderr) + continue + arg = call.args[0] + if not isinstance(arg, ast.Constant): + print(_( + '*** %(file)s:%(lineno)s: Seen unexpected argument type' + ' in gettext call: %(source_segment)s' + ) % { + 'source_segment': ast.get_source_segment(tstring, call) or tstring, + 'file': self.__curfile, + 'lineno': lineno + }, file=sys.stderr) + continue + if isinstance(arg.value, str): + self.__addentry(arg.value, lineno) def __suiteseen(self, ttype, tstring, lineno): # skip over any enclosure pairs until we see the colon From 38811d68caf9b782ea7168479acb09557e126efe Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Tue, 10 Nov 2020 14:57:34 +0900 Subject: [PATCH 55/61] Fix typo in unicodeobject.c (GH-23180) exeeds -> exceeds Automerge-Triggered-By: GH:Mariatta --- Objects/unicodeobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 90580182010..e7a63e7b973 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -1894,7 +1894,7 @@ _PyUnicode_Ready(PyObject *unicode) _PyUnicode_WSTR_LENGTH(unicode) = 0; #endif } - /* maxchar exeeds 16 bit, wee need 4 bytes for unicode characters */ + /* maxchar exceeds 16 bit, wee need 4 bytes for unicode characters */ else { #if SIZEOF_WCHAR_T == 2 /* in case the native representation is 2-bytes, we need to allocate a From 9e1b828265e6bfb58f1e0299bd78d8ff6347a2ba Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 10 Nov 2020 13:21:52 +0100 Subject: [PATCH 56/61] bpo-42260: Compute the path config in the main init (GH-23211) The path configuration is now computed in the "main" initialization. The core initialization no longer computes it. * Add _PyConfig_Read() function to read the configuration without computing the path configuration. * pyinit_core() no longer computes the path configuration: it is now computed by init_interp_main(). * The path configuration output members of PyConfig are now optional: * executable * base_executable * prefix * base_prefix * exec_prefix * base_exec_prefix * _PySys_UpdateConfig() now skips NULL strings in PyConfig. * _testembed: Rename test_set_config() to test_init_set_config() for consistency with other tests. --- Include/internal/pycore_initconfig.h | 1 + Lib/test/_test_embed_set_config.py | 26 ++++++++---- Programs/_testembed.c | 4 +- Python/initconfig.c | 47 +++++++++------------- Python/pylifecycle.c | 59 +++++++++++++++------------- Python/sysmodule.c | 22 +++++++---- 6 files changed, 85 insertions(+), 74 deletions(-) diff --git a/Include/internal/pycore_initconfig.h b/Include/internal/pycore_initconfig.h index 325be5494d4..d8400b1c42e 100644 --- a/Include/internal/pycore_initconfig.h +++ b/Include/internal/pycore_initconfig.h @@ -152,6 +152,7 @@ extern PyStatus _PyConfig_Copy( PyConfig *config, const PyConfig *config2); extern PyStatus _PyConfig_InitPathConfig(PyConfig *config); +extern PyStatus _PyConfig_Read(PyConfig *config, int compute_path_config); extern PyStatus _PyConfig_Write(const PyConfig *config, struct pyruntimestate *runtime); extern PyStatus _PyConfig_SetPyArgv( diff --git a/Lib/test/_test_embed_set_config.py b/Lib/test/_test_embed_set_config.py index a19f8db1584..82c5d829823 100644 --- a/Lib/test/_test_embed_set_config.py +++ b/Lib/test/_test_embed_set_config.py @@ -100,19 +100,19 @@ class SetConfigTests(unittest.TestCase): 'check_hash_pycs_mode', 'program_name', 'platlibdir', - 'executable', - 'base_executable', - 'prefix', - 'base_prefix', - 'exec_prefix', - 'base_exec_prefix', # optional wstr: # 'pythonpath_env' - # 'home', + # 'home' # 'pycache_prefix' # 'run_command' # 'run_module' # 'run_filename' + # 'executable' + # 'prefix' + # 'exec_prefix' + # 'base_executable' + # 'base_prefix' + # 'base_exec_prefix' ): value_tests.append((key, invalid_wstr)) type_tests.append((key, b'bytes')) @@ -217,6 +217,18 @@ class SetConfigTests(unittest.TestCase): self.set_config(base_executable="base_executable") self.assertEqual(sys._base_executable, "base_executable") + # When base_xxx is NULL, value is copied from xxxx + self.set_config( + executable='executable', + prefix="prefix", + exec_prefix="exec_prefix", + base_executable=None, + base_prefix=None, + base_exec_prefix=None) + self.assertEqual(sys._base_executable, "executable") + self.assertEqual(sys.base_prefix, "prefix") + self.assertEqual(sys.base_exec_prefix, "exec_prefix") + def test_path(self): self.set_config(module_search_paths_set=1, module_search_paths=['a', 'b', 'c']) diff --git a/Programs/_testembed.c b/Programs/_testembed.c index cb3a23a101e..748ea8a8f33 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -1547,7 +1547,7 @@ static int tune_config(void) } -static int test_set_config(void) +static int test_init_set_config(void) { // Initialize core PyConfig config; @@ -1742,7 +1742,7 @@ static struct TestCase TestCases[] = { {"test_init_setpath_config", test_init_setpath_config}, {"test_init_setpythonhome", test_init_setpythonhome}, {"test_init_warnoptions", test_init_warnoptions}, - {"test_init_set_config", test_set_config}, + {"test_init_set_config", test_init_set_config}, {"test_run_main", test_run_main}, {"test_get_argc_argv", test_get_argc_argv}, diff --git a/Python/initconfig.c b/Python/initconfig.c index e0811b56cb3..11db4a3ef59 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -619,15 +619,6 @@ config_check_consistency(const PyConfig *config) assert(_PyWideStringList_CheckConsistency(&config->warnoptions)); assert(_PyWideStringList_CheckConsistency(&config->module_search_paths)); assert(config->module_search_paths_set >= 0); - if (config->_install_importlib) { - /* don't check config->module_search_paths */ - assert(config->executable != NULL); - assert(config->base_executable != NULL); - assert(config->prefix != NULL); - assert(config->base_prefix != NULL); - assert(config->exec_prefix != NULL); - assert(config->base_exec_prefix != NULL); - } assert(config->platlibdir != NULL); assert(config->filesystem_encoding != NULL); assert(config->filesystem_errors != NULL); @@ -1297,24 +1288,15 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict) GET_WSTR_OPT(home); GET_WSTR(platlibdir); + // Path configuration output GET_UINT(module_search_paths_set); GET_WSTRLIST(module_search_paths); - if (config->_install_importlib) { - GET_WSTR(executable); - GET_WSTR(base_executable); - GET_WSTR(prefix); - GET_WSTR(base_prefix); - GET_WSTR(exec_prefix); - GET_WSTR(base_exec_prefix); - } - else { - GET_WSTR_OPT(executable); - GET_WSTR_OPT(base_executable); - GET_WSTR_OPT(prefix); - GET_WSTR_OPT(base_prefix); - GET_WSTR_OPT(exec_prefix); - GET_WSTR_OPT(base_exec_prefix); - } + GET_WSTR_OPT(executable); + GET_WSTR_OPT(base_executable); + GET_WSTR_OPT(prefix); + GET_WSTR_OPT(base_prefix); + GET_WSTR_OPT(exec_prefix); + GET_WSTR_OPT(base_exec_prefix); GET_UINT(skip_source_first_line); GET_WSTR_OPT(run_command); @@ -2043,7 +2025,7 @@ config_init_fs_encoding(PyConfig *config, const PyPreConfig *preconfig) static PyStatus -config_read(PyConfig *config) +config_read(PyConfig *config, int compute_path_config) { PyStatus status; const PyPreConfig *preconfig = &_PyRuntime.preconfig; @@ -2087,7 +2069,7 @@ config_read(PyConfig *config) } } - if (config->_install_importlib) { + if (compute_path_config && config->_install_importlib) { status = _PyConfig_InitPathConfig(config); if (_PyStatus_EXCEPTION(status)) { return status; @@ -2834,7 +2816,7 @@ PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, The only side effects are to modify config and to call _Py_SetArgcArgv(). */ PyStatus -PyConfig_Read(PyConfig *config) +_PyConfig_Read(PyConfig *config, int compute_path_config) { PyStatus status; @@ -2877,7 +2859,7 @@ PyConfig_Read(PyConfig *config) goto done; } - status = config_read(config); + status = config_read(config, compute_path_config); if (_PyStatus_EXCEPTION(status)) { goto done; } @@ -2892,6 +2874,13 @@ done: } +PyStatus +PyConfig_Read(PyConfig *config) +{ + return _PyConfig_Read(config, 1); +} + + PyObject* _Py_GetConfigsAsDict(void) { diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index e34d6471e17..609e0a42e4d 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -429,25 +429,20 @@ _Py_SetLocaleFromEnv(int category) static int -interpreter_set_config(const PyConfig *config) +interpreter_update_config(PyThreadState *tstate, int only_update_path_config) { - PyThreadState *tstate = PyThreadState_Get(); + const PyConfig *config = &tstate->interp->config; - PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - return -1; + if (!only_update_path_config) { + PyStatus status = _PyConfig_Write(config, tstate->interp->runtime); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + return -1; + } } - status = _PyConfig_Copy(&tstate->interp->config, config); - if (_PyStatus_EXCEPTION(status)) { - _PyErr_SetFromPyStatus(status); - return -1; - } - config = &tstate->interp->config; - - if (config->_install_importlib && _Py_IsMainInterpreter(tstate)) { - status = _PyConfig_WritePathConfig(config); + if (_Py_IsMainInterpreter(tstate)) { + PyStatus status = _PyConfig_WritePathConfig(config); if (_PyStatus_EXCEPTION(status)) { _PyErr_SetFromPyStatus(status); return -1; @@ -465,6 +460,7 @@ interpreter_set_config(const PyConfig *config) int _PyInterpreterState_SetConfig(const PyConfig *src_config) { + PyThreadState *tstate = PyThreadState_Get(); int res = -1; PyConfig config; @@ -481,7 +477,13 @@ _PyInterpreterState_SetConfig(const PyConfig *src_config) goto done; } - res = interpreter_set_config(&config); + status = _PyConfig_Copy(&tstate->interp->config, &config); + if (_PyStatus_EXCEPTION(status)) { + _PyErr_SetFromPyStatus(status); + goto done; + } + + res = interpreter_update_config(tstate, 0); done: PyConfig_Clear(&config); @@ -763,13 +765,6 @@ pycore_init_import_warnings(PyThreadState *tstate, PyObject *sysmod) const PyConfig *config = _PyInterpreterState_GetConfig(tstate->interp); if (config->_install_importlib) { - if (_Py_IsMainInterpreter(tstate)) { - status = _PyConfig_WritePathConfig(config); - if (_PyStatus_EXCEPTION(status)) { - return status; - } - } - /* This call sets up builtin and frozen import support */ status = init_importlib(tstate, sysmod); if (_PyStatus_EXCEPTION(status)) { @@ -985,7 +980,9 @@ pyinit_core(_PyRuntimeState *runtime, goto done; } - status = PyConfig_Read(&config); + // Read the configuration, but don't compute the path configuration + // (it is computed in the main init). + status = _PyConfig_Read(&config, 0); if (_PyStatus_EXCEPTION(status)) { goto done; } @@ -1012,8 +1009,8 @@ done: static PyStatus pyinit_main_reconfigure(PyThreadState *tstate) { - if (_PySys_UpdateConfig(tstate) < 0) { - return _PyStatus_ERR("fail to update sys for the new conf"); + if (interpreter_update_config(tstate, 0) < 0) { + return _PyStatus_ERR("fail to reconfigure Python"); } return _PyStatus_OK(); } @@ -1041,14 +1038,20 @@ init_interp_main(PyThreadState *tstate) return _PyStatus_OK(); } + // Compute the path configuration + status = _PyConfig_InitPathConfig(&interp->config); + if (_PyStatus_EXCEPTION(status)) { + return status; + } + if (is_main_interp) { if (_PyTime_Init() < 0) { return _PyStatus_ERR("can't initialize time"); } } - if (_PySys_UpdateConfig(tstate) < 0) { - return _PyStatus_ERR("can't finish initializing sys"); + if (interpreter_update_config(tstate, 1) < 0) { + return _PyStatus_ERR("failed to update the Python config"); } status = init_importlib_external(tstate); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index ae4f0eeb2ee..61741f7432d 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -2922,17 +2922,22 @@ _PySys_UpdateConfig(PyThreadState *tstate) #define SET_SYS_FROM_WSTR(KEY, VALUE) \ SET_SYS(KEY, PyUnicode_FromWideChar(VALUE, -1)); +#define COPY_WSTR(SYS_ATTR, WSTR) \ + if (WSTR != NULL) { \ + SET_SYS_FROM_WSTR(SYS_ATTR, WSTR); \ + } + if (config->module_search_paths_set) { COPY_LIST("path", config->module_search_paths); } - SET_SYS_FROM_WSTR("executable", config->executable); - SET_SYS_FROM_WSTR("_base_executable", config->base_executable); - SET_SYS_FROM_WSTR("prefix", config->prefix); - SET_SYS_FROM_WSTR("base_prefix", config->base_prefix); - SET_SYS_FROM_WSTR("exec_prefix", config->exec_prefix); - SET_SYS_FROM_WSTR("base_exec_prefix", config->base_exec_prefix); - SET_SYS_FROM_WSTR("platlibdir", config->platlibdir); + COPY_WSTR("executable", config->executable); + COPY_WSTR("_base_executable", config->base_executable); + COPY_WSTR("prefix", config->prefix); + COPY_WSTR("base_prefix", config->base_prefix); + COPY_WSTR("exec_prefix", config->exec_prefix); + COPY_WSTR("base_exec_prefix", config->base_exec_prefix); + COPY_WSTR("platlibdir", config->platlibdir); if (config->pycache_prefix != NULL) { SET_SYS_FROM_WSTR("pycache_prefix", config->pycache_prefix); @@ -2946,8 +2951,9 @@ _PySys_UpdateConfig(PyThreadState *tstate) SET_SYS("_xoptions", sys_create_xoptions_dict(config)); -#undef COPY_LIST #undef SET_SYS_FROM_WSTR +#undef COPY_LIST +#undef COPY_WSTR // sys.flags PyObject *flags = _PySys_GetObject(tstate, "flags"); // borrowed ref From 9303b9ecc39e4e79e34d6ce7258036a9d0876046 Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Tue, 10 Nov 2020 21:42:36 +0900 Subject: [PATCH 57/61] Fix typo in test_array.py (GH-23189) --- Lib/test/test_array.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index c423f545488..77a0c64a88e 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -393,7 +393,7 @@ class BaseTest: a.insert(0, self.outside) self.assertEqual(list(exhit), []) # The iterator index points past the 0th position so inserting - # an element in the beggining does not make it appear. + # an element in the beginning does not make it appear. self.assertEqual(list(empit), []) self.assertEqual(list(a), [self.outside] + list(self.example)) From 0b9c4c6fcf2b0673fa45ddfa092934a9d5479b8c Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 10 Nov 2020 14:47:31 +0100 Subject: [PATCH 58/61] bpo-42171: Add PEP573-related items to the limited API (GH-23009) --- Doc/c-api/structures.rst | 6 ++++-- Include/methodobject.h | 2 +- .../next/C API/2020-10-27-21-10-14.bpo-42171.S3FWTP.rst | 4 ++++ PC/python3dll.c | 4 ++++ 4 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 Misc/NEWS.d/next/C API/2020-10-27-21-10-14.bpo-42171.S3FWTP.rst diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst index 03fe479165f..37602ed5b4d 100644 --- a/Doc/c-api/structures.rst +++ b/Doc/c-api/structures.rst @@ -263,10 +263,12 @@ There are these calling conventions: of :c:type:`PyObject*` values indicating the arguments and the third parameter is the number of arguments (the length of the array). - This is not part of the :ref:`limited API `. - .. versionadded:: 3.7 + .. versionchanged:: 3.10 + + ``METH_FASTCALL`` is now part of the stable ABI. + .. data:: METH_FASTCALL | METH_KEYWORDS diff --git a/Include/methodobject.h b/Include/methodobject.h index 12e049b4043..5d06d7691ba 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -73,7 +73,7 @@ PyAPI_FUNC(PyObject *) PyCMethod_New(PyMethodDef *, PyObject *, #define METH_COEXIST 0x0040 -#ifndef Py_LIMITED_API +#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03100000 #define METH_FASTCALL 0x0080 #endif diff --git a/Misc/NEWS.d/next/C API/2020-10-27-21-10-14.bpo-42171.S3FWTP.rst b/Misc/NEWS.d/next/C API/2020-10-27-21-10-14.bpo-42171.S3FWTP.rst new file mode 100644 index 00000000000..5dfbb23a6a3 --- /dev/null +++ b/Misc/NEWS.d/next/C API/2020-10-27-21-10-14.bpo-42171.S3FWTP.rst @@ -0,0 +1,4 @@ +The :c:data:`METH_FASTCALL` calling convention is added to the limited API. +The functions :c:func:`PyModule_AddType`, :c:func:`PyType_FromModuleAndSpec`, +:c:func:`PyType_GetModule` and :c:func:`PyType_GetModuleState` are added to +the limited API on Windows. diff --git a/PC/python3dll.c b/PC/python3dll.c index d1fdd0ac54c..27cc315de2d 100644 --- a/PC/python3dll.c +++ b/PC/python3dll.c @@ -344,6 +344,7 @@ EXPORT_FUNC(PyModule_AddFunctions) EXPORT_FUNC(PyModule_AddIntConstant) EXPORT_FUNC(PyModule_AddObject) EXPORT_FUNC(PyModule_AddStringConstant) +EXPORT_FUNC(PyModule_AddType) EXPORT_FUNC(PyModule_Create2) EXPORT_FUNC(PyModule_ExecDef) EXPORT_FUNC(PyModule_FromDefAndSpec2) @@ -547,9 +548,12 @@ EXPORT_FUNC(PyTuple_Size) EXPORT_FUNC(PyType_ClearCache) EXPORT_FUNC(PyType_FromSpec) EXPORT_FUNC(PyType_FromSpecWithBases) +EXPORT_FUNC(PyType_FromModuleAndSpec) EXPORT_FUNC(PyType_GenericAlloc) EXPORT_FUNC(PyType_GenericNew) EXPORT_FUNC(PyType_GetFlags) +EXPORT_FUNC(PyType_GetModule) +EXPORT_FUNC(PyType_GetModuleState) EXPORT_FUNC(PyType_GetSlot) EXPORT_FUNC(PyType_IsSubtype) EXPORT_FUNC(PyType_Modified) From 42d873c63aa9d160c132be4a34599531574db12c Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Tue, 10 Nov 2020 15:58:31 +0200 Subject: [PATCH 59/61] bpo-42183: Fix a stack overflow error for asyncio Task or Future repr() (GH-23020) The overflow occurs under some circumstances when a task or future recursively returns itself. Co-authored-by: Kyle Stanley --- Lib/asyncio/base_futures.py | 25 ++++++++++++++++--- Lib/test/test_asyncio/test_futures2.py | 18 +++++++++++++ .../2020-10-29-11-17-35.bpo-42183.50ZcIi.rst | 4 +++ 3 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 Lib/test/test_asyncio/test_futures2.py create mode 100644 Misc/NEWS.d/next/Library/2020-10-29-11-17-35.bpo-42183.50ZcIi.rst diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 22f298069c5..2c01ac98e10 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -1,6 +1,7 @@ __all__ = () import reprlib +from _thread import get_ident from . import format_helpers @@ -41,6 +42,16 @@ def _format_callbacks(cb): return f'cb=[{cb}]' +# bpo-42183: _repr_running is needed for repr protection +# when a Future or Task result contains itself directly or indirectly. +# The logic is borrowed from @reprlib.recursive_repr decorator. +# Unfortunately, the direct decorator usage is impossible because of +# AttributeError: '_asyncio.Task' object has no attribute '__module__' error. +# +# After fixing this thing we can return to the decorator based approach. +_repr_running = set() + + def _future_repr_info(future): # (Future) -> str """helper function for Future.__repr__""" @@ -49,9 +60,17 @@ def _future_repr_info(future): if future._exception is not None: info.append(f'exception={future._exception!r}') else: - # use reprlib to limit the length of the output, especially - # for very long strings - result = reprlib.repr(future._result) + key = id(future), get_ident() + if key in _repr_running: + result = '...' + else: + _repr_running.add(key) + try: + # use reprlib to limit the length of the output, especially + # for very long strings + result = reprlib.repr(future._result) + finally: + _repr_running.discard(key) info.append(f'result={result}') if future._callbacks: info.append(_format_callbacks(future._callbacks)) diff --git a/Lib/test/test_asyncio/test_futures2.py b/Lib/test/test_asyncio/test_futures2.py new file mode 100644 index 00000000000..13dbc703277 --- /dev/null +++ b/Lib/test/test_asyncio/test_futures2.py @@ -0,0 +1,18 @@ +# IsolatedAsyncioTestCase based tests +import asyncio +import unittest + + +class FutureTests(unittest.IsolatedAsyncioTestCase): + async def test_recursive_repr_for_pending_tasks(self): + # The call crashes if the guard for recursive call + # in base_futures:_future_repr_info is absent + # See Also: https://bugs.python.org/issue42183 + + async def func(): + return asyncio.all_tasks() + + # The repr() call should not raise RecursiveError at first. + # The check for returned string is not very reliable but + # exact comparison for the whole string is even weaker. + self.assertIn('...', repr(await asyncio.wait_for(func(), timeout=10))) diff --git a/Misc/NEWS.d/next/Library/2020-10-29-11-17-35.bpo-42183.50ZcIi.rst b/Misc/NEWS.d/next/Library/2020-10-29-11-17-35.bpo-42183.50ZcIi.rst new file mode 100644 index 00000000000..f6d7653f2cf --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-10-29-11-17-35.bpo-42183.50ZcIi.rst @@ -0,0 +1,4 @@ +Fix a stack overflow error for asyncio Task or Future repr(). + +The overflow occurs under some circumstances when a Task or Future +recursively returns itself. From 3eb284622579d04289d0e3b2e97fbf56df6e845e Mon Sep 17 00:00:00 2001 From: Joongi Kim Date: Wed, 11 Nov 2020 00:19:11 +0900 Subject: [PATCH 60/61] Update whatsnew for 3.10 release about addition of contextlib.aclosing (GH-23217) --- Doc/whatsnew/3.10.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/whatsnew/3.10.rst b/Doc/whatsnew/3.10.rst index a735bf23543..95cdb06f997 100644 --- a/Doc/whatsnew/3.10.rst +++ b/Doc/whatsnew/3.10.rst @@ -176,6 +176,13 @@ codecs Add a :func:`codecs.unregister` function to unregister a codec search function. (Contributed by Hai Shi in :issue:`41842`.) +contextlib +---------- + +Add a :func:`contextlib.aclosing` context manager to safely close async generators +and objects representing asynchronously released resources. +(Contributed by Joongi Kim and John Belmonte in :issue:`41229`.) + curses ------ From e59b2deffde61e5641cabd65034fa11b4db898ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Michal=20=C4=8Ciha=C5=99?= Date: Tue, 10 Nov 2020 17:06:02 +0100 Subject: [PATCH 61/61] bpo-42014: shutil.rmtree: call onerror with correct function (GH-22585) The onerror is supposed to be called with failed function, but in this case lstat is wrongly used instead of open. Not sure if this needs bug or not... Automerge-Triggered-By: GH:hynek --- Lib/shutil.py | 2 +- .../next/Library/2020-11-10-15-40-56.bpo-42014.ShM37l.rst | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 Misc/NEWS.d/next/Library/2020-11-10-15-40-56.bpo-42014.ShM37l.rst diff --git a/Lib/shutil.py b/Lib/shutil.py index 223e9a8a705..f0e833dc979 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -711,7 +711,7 @@ def rmtree(path, ignore_errors=False, onerror=None): try: fd = os.open(path, os.O_RDONLY) except Exception: - onerror(os.lstat, path, sys.exc_info()) + onerror(os.open, path, sys.exc_info()) return try: if os.path.samestat(orig_st, os.fstat(fd)): diff --git a/Misc/NEWS.d/next/Library/2020-11-10-15-40-56.bpo-42014.ShM37l.rst b/Misc/NEWS.d/next/Library/2020-11-10-15-40-56.bpo-42014.ShM37l.rst new file mode 100644 index 00000000000..d3e1abcd84c --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-11-10-15-40-56.bpo-42014.ShM37l.rst @@ -0,0 +1 @@ +The ``onerror`` callback from ``shutil.rmtree`` now receives correct function when ``os.open`` fails. \ No newline at end of file