From 267862f9ca3369af3a43148368bb853fbd897798 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Wed, 7 Dec 2016 16:19:56 -0800 Subject: [PATCH 01/14] Issue #28635: Drop the note that whatsnew is incomplete (grafted from d12bc674b74eee73365e38fad1f170ed3349bd59) --- Doc/whatsnew/3.6.rst | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 3c2ab129762..527e7429b1e 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -46,12 +46,6 @@ This saves the maintainer the effort of going through the Mercurial log when researching a change. -.. note:: - - Prerelease users should be aware that this document is currently in draft - form. It will be updated substantially as Python 3.6 moves towards release, - so it's worth checking back even after reading earlier versions. - This article explains the new features in Python 3.6, compared to 3.5. .. seealso:: @@ -62,9 +56,6 @@ This article explains the new features in Python 3.6, compared to 3.5. Summary -- Release highlights ============================= -.. This section singles out the most important changes in Python 3.6. - Brevity is key. - New syntax features: * :ref:`PEP 498 `, formatted string literals. From 446e606bfe9d2398ad9dfd538085e36afb6c3384 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Wed, 7 Dec 2016 23:37:12 -0500 Subject: [PATCH 02/14] Issue #28900: Update documentation sidebar for 3.6.0rc. --- Doc/tools/templates/indexsidebar.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html index bb449ef1cd3..413c0a7699c 100644 --- a/Doc/tools/templates/indexsidebar.html +++ b/Doc/tools/templates/indexsidebar.html @@ -3,7 +3,8 @@

{% trans %}Docs for other versions{% endtrans %}

From b5e688a8ff37d4fff06c06d6ec625ac6653306f6 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 7 Dec 2016 23:54:28 -0800 Subject: [PATCH 03/14] guard HAVE_LONG_LONG definition to prevent redefinition (#28898) (grafted from 4745d801cae2d57e3432313acd0b76b8b4cc9c75) --- Include/pyport.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Include/pyport.h b/Include/pyport.h index 20f3db74815..28bf4b21485 100644 --- a/Include/pyport.h +++ b/Include/pyport.h @@ -37,9 +37,10 @@ Used in: Py_SAFE_DOWNCAST * integral synonyms. Only define the ones we actually need. */ -// long long is required now. Define HAVE_LONG_LONG unconditionally for -// compatibility. +// long long is required. Ensure HAVE_LONG_LONG is defined for compatibility. +#ifndef HAVE_LONG_LONG #define HAVE_LONG_LONG +#endif #ifndef PY_LONG_LONG #define PY_LONG_LONG long long /* If LLONG_MAX is defined in limits.h, use that. */ From 5dc1fb3eda58fd01b47e7b3bba7ce1dff226054a Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Wed, 7 Dec 2016 13:02:27 -0800 Subject: [PATCH 04/14] Issue #28896: Deprecate WindowsRegistryFinder (grafted from 25df9671663b5f8b1560d58d8842f9676f6dffc2) --- Doc/library/importlib.rst | 4 ++++ Doc/using/windows.rst | 8 ++++++++ Doc/whatsnew/3.6.rst | 4 ++++ Misc/NEWS | 11 +++++++++++ 4 files changed, 27 insertions(+) diff --git a/Doc/library/importlib.rst b/Doc/library/importlib.rst index 8210a2f1709..1fd56983d0a 100644 --- a/Doc/library/importlib.rst +++ b/Doc/library/importlib.rst @@ -806,6 +806,10 @@ find and load modules. .. versionadded:: 3.3 + .. deprecated:: 3.6 + Use :mod:`site` configuration instead. Future versions of Python may + not enable this finder by default. + .. class:: PathFinder diff --git a/Doc/using/windows.rst b/Doc/using/windows.rst index 81efbb0c985..3e4b70e8a17 100644 --- a/Doc/using/windows.rst +++ b/Doc/using/windows.rst @@ -823,6 +823,14 @@ non-standard paths in the registry and user site-packages. * Adds ``pythonXX.zip`` as a potential landmark when directly adjacent to the executable. +.. deprecated:: + 3.6 + + Modules specified in the registry under ``Modules`` (not ``PythonPath``) + may be imported by :class:`importlib.machinery.WindowsRegistryFinder`. + This finder is enabled on Windows in 3.6.0 and earlier, but may need to + be explicitly added to :attr:`sys.meta_path` in the future. + Additional modules ================== diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 527e7429b1e..ddca2ef8ca3 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -1938,6 +1938,10 @@ are now deprecated. They were the only remaining implementations of been deprecated in previous versions of Python in favour of :meth:`importlib.abc.Loader.exec_module`. +The :class:`importlib.machinery.WindowsRegistryFinder` class is now +deprecated. As of 3.6.0, it is still added to :attr:`sys.meta_path` by +default (on Windows), but this may change in future releases. + os ~~ diff --git a/Misc/NEWS b/Misc/NEWS index 770c8a9623f..bab6558e12a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -2,6 +2,17 @@ Python News +++++++++++ +What's New in Python 3.6.0 release candidate 2 +============================================== + +*Release date: XXXX-XX-XX* + +Windows +------- + +- Issue #28896: Deprecate WindowsRegistryFinder + + What's New in Python 3.6.0 release candidate 1 ============================================== From 2308740f64d4de603a7a818e718189e933625831 Mon Sep 17 00:00:00 2001 From: Steve Dower Date: Sun, 11 Dec 2016 14:48:32 -0800 Subject: [PATCH 05/14] Issue #28783: Replaces bdist_wininst in nuget packages with stub --- Tools/msi/distutils.command.__init__.py | 32 -------------------- Tools/msi/distutils.command.bdist_wininst.py | 20 ++++++++++++ Tools/msi/make_zip.py | 8 ++--- 3 files changed, 22 insertions(+), 38 deletions(-) delete mode 100644 Tools/msi/distutils.command.__init__.py create mode 100644 Tools/msi/distutils.command.bdist_wininst.py diff --git a/Tools/msi/distutils.command.__init__.py b/Tools/msi/distutils.command.__init__.py deleted file mode 100644 index 83f34b470ab..00000000000 --- a/Tools/msi/distutils.command.__init__.py +++ /dev/null @@ -1,32 +0,0 @@ -"""distutils.command - -Package containing implementation of all the standard Distutils -commands.""" - -__all__ = ['build', - 'build_py', - 'build_ext', - 'build_clib', - 'build_scripts', - 'clean', - 'install', - 'install_lib', - 'install_headers', - 'install_scripts', - 'install_data', - 'sdist', - 'register', - 'bdist', - 'bdist_dumb', - 'bdist_rpm', - # This command is not included in this package - #'bdist_wininst', - 'check', - 'upload', - # These two are reserved for future use: - #'bdist_sdux', - #'bdist_pkgtool', - # Note: - # bdist_packager is not included because it only provides - # an abstract base class - ] diff --git a/Tools/msi/distutils.command.bdist_wininst.py b/Tools/msi/distutils.command.bdist_wininst.py new file mode 100644 index 00000000000..d586e34fec8 --- /dev/null +++ b/Tools/msi/distutils.command.bdist_wininst.py @@ -0,0 +1,20 @@ +"""distutils.command.bdist_wininst + +Suppresses the 'bdist_wininst' command, while still allowing +setuptools to import it without breaking.""" + +from distutils.core import Command +from distutils.errors import DistutilsPlatformError + +class bdist_wininst(Command): + description = "create an executable installer for MS Windows" + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + raise DistutilsPlatformError("bdist_wininst is not supported " + "in this Python distribution") diff --git a/Tools/msi/make_zip.py b/Tools/msi/make_zip.py index 09f6fe328f3..8dbe83e4f44 100644 --- a/Tools/msi/make_zip.py +++ b/Tools/msi/make_zip.py @@ -79,10 +79,6 @@ def include_in_lib(p): if name in EXCLUDE_FILE_FROM_LIBRARY: return False - # Special code is included below to patch this file back in - if [d.lower() for d in p.parts[-3:]] == ['distutils', 'command', '__init__.py']: - return False - suffix = p.suffix.lower() return suffix not in {'.pyc', '.pyo', '.exe'} @@ -218,8 +214,8 @@ def main(): extra_files = [] if s == 'Lib' and p == '**/*': extra_files.append(( - source / 'tools' / 'msi' / 'distutils.command.__init__.py', - Path('distutils') / 'command' / '__init__.py' + source / 'tools' / 'msi' / 'distutils.command.bdist_wininst.py', + Path('distutils') / 'command' / 'bdist_wininst.py' )) copied = copy_to_layout(temp / t.rstrip('/'), chain(files, extra_files)) print('Copied {} files'.format(copied)) From 291b93bb1f14257dcd0095bad840e15d4b7dd77e Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Mon, 12 Dec 2016 16:44:58 -0500 Subject: [PATCH 06/14] Issue #28089: Document TCP_NODELAY in asyncio Initial patch by Mariatta Wijaya. (grafted from 853e3f4d6cd98ac4590238bc1c60e40fd8ed3895) --- Doc/library/asyncio-protocol.rst | 3 +++ Doc/whatsnew/3.6.rst | 3 +++ 2 files changed, 6 insertions(+) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst index c0342f7f2f3..3fbf5105867 100644 --- a/Doc/library/asyncio-protocol.rst +++ b/Doc/library/asyncio-protocol.rst @@ -25,6 +25,9 @@ the transport's kind. The transport classes are :ref:`not thread safe `. +.. versionchanged:: 3.6 + The socket option ``TCP_NODELAY`` is now set by default. + BaseTransport ------------- diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index ddca2ef8ca3..0187f94c6f4 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -875,6 +875,9 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 but that use asyncio to handle them. (Contributed by Jim Fulton in :issue:`27392`.) +* ``TCP_NODELAY`` flag is now set for all TCP transports by default. + (Contributed by Yury Selivanov in :issue:`27456`.) + binascii -------- From 5e65a5f4efeca3b656779973c465abe756e86a67 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 15 Dec 2016 16:20:53 +0100 Subject: [PATCH 07/14] Issue #28979: Fix What's New in Python 3.6, dict The new dict implementation is not faster, but more compact. Patch written by Brendan Donegan. (grafted from 181453f9a0c424212f0f6ddca2b9065c15689d7c) --- Doc/whatsnew/3.6.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 0187f94c6f4..84c452a0493 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -77,7 +77,7 @@ New library modules: CPython implementation improvements: * The :ref:`dict ` type has been reimplemented to use - a :ref:`faster, more compact representation ` + a :ref:`more compact representation ` similar to the `PyPy dict implementation`_. This resulted in dictionaries using 20% to 25% less memory when compared to Python 3.5. From ccda5c41df456ce6b6b40198233b83e1b18b78db Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 15 Dec 2016 17:21:23 +0100 Subject: [PATCH 08/14] Fix a memory leak in split-table dictionaries Issue #28147: Fix a memory leak in split-table dictionaries: setattr() must not convert combined table into split table. Patch written by INADA Naoki. (grafted from 85be9dcc16a81d3ccd1f67b056255a7f206edd47) --- Lib/test/test_dict.py | 30 ++++++++++++++++++++++++++++++ Misc/NEWS | 7 +++++++ Modules/_testcapimodule.c | 14 ++++++++++++++ Objects/dictobject.c | 27 ++++++++++++++++++++++----- 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index cd077ff0e0c..832bb9c8e2d 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -933,6 +933,36 @@ class DictTest(unittest.TestCase): self.assertEqual(list(a), ['x', 'y']) self.assertEqual(list(b), ['x', 'y', 'z']) + @support.cpython_only + def test_splittable_setattr_after_pop(self): + """setattr() must not convert combined table into split table.""" + # Issue 28147 + import _testcapi + + class C: + pass + a = C() + + a.a = 1 + self.assertTrue(_testcapi.dict_hassplittable(a.__dict__)) + + # dict.pop() convert it to combined table + a.__dict__.pop('a') + self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) + + # But C should not convert a.__dict__ to split table again. + a.a = 1 + self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) + + # Same for popitem() + a = C() + a.a = 2 + self.assertTrue(_testcapi.dict_hassplittable(a.__dict__)) + a.__dict__.popitem() + self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) + a.a = 3 + self.assertFalse(_testcapi.dict_hassplittable(a.__dict__)) + def test_iterator_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): data = {1:"a", 2:"b", 3:"c"} diff --git a/Misc/NEWS b/Misc/NEWS index bab6558e12a..3e1517939a4 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -7,6 +7,13 @@ What's New in Python 3.6.0 release candidate 2 *Release date: XXXX-XX-XX* +Core and Builtins +----------------- + +- Issue #28147: Fix a memory leak in split-table dictionaries: setattr() + must not convert combined table into split table. Patch written by INADA + Naoki. + Windows ------- diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index ecfc0858c14..f09205f63cc 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -259,6 +259,19 @@ dict_getitem_knownhash(PyObject *self, PyObject *args) return result; } +static PyObject* +dict_hassplittable(PyObject *self, PyObject *arg) +{ + if (!PyDict_Check(arg)) { + PyErr_Format(PyExc_TypeError, + "dict_hassplittable() argument must be dict, not '%s'", + arg->ob_type->tp_name); + return NULL; + } + + return PyBool_FromLong(_PyDict_HasSplitTable((PyDictObject*)arg)); +} + /* Issue #4701: Check that PyObject_Hash implicitly calls * PyType_Ready if it hasn't already been called */ @@ -4024,6 +4037,7 @@ static PyMethodDef TestMethods[] = { {"test_list_api", (PyCFunction)test_list_api, METH_NOARGS}, {"test_dict_iteration", (PyCFunction)test_dict_iteration,METH_NOARGS}, {"dict_getitem_knownhash", dict_getitem_knownhash, METH_VARARGS}, + {"dict_hassplittable", dict_hassplittable, METH_O}, {"test_lazy_hash_inheritance", (PyCFunction)test_lazy_hash_inheritance,METH_NOARGS}, {"test_long_api", (PyCFunction)test_long_api, METH_NOARGS}, {"test_xincref_doesnt_leak",(PyCFunction)test_xincref_doesnt_leak, METH_NOARGS}, diff --git a/Objects/dictobject.c b/Objects/dictobject.c index d8ab91fb12d..20b6f2f52e4 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -1245,7 +1245,7 @@ After resizing a table is always combined, but can be resplit by make_keys_shared(). */ static int -dictresize(PyDictObject *mp, Py_ssize_t minused) +dictresize(PyDictObject *mp, Py_ssize_t minsize) { Py_ssize_t i, newsize; PyDictKeysObject *oldkeys; @@ -1254,7 +1254,7 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) /* Find the smallest table size > minused. */ for (newsize = PyDict_MINSIZE; - newsize <= minused && newsize > 0; + newsize < minsize && newsize > 0; newsize <<= 1) ; if (newsize <= 0) { @@ -1269,6 +1269,8 @@ dictresize(PyDictObject *mp, Py_ssize_t minused) mp->ma_keys = oldkeys; return -1; } + // New table must be large enough. + assert(mp->ma_keys->dk_usable >= mp->ma_used); if (oldkeys->dk_lookup == lookdict) mp->ma_keys->dk_lookup = lookdict; mp->ma_values = NULL; @@ -4292,10 +4294,25 @@ _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, CACHED_KEYS(tp) = NULL; DK_DECREF(cached); } - } else { + } + else { + int was_shared = cached == ((PyDictObject *)dict)->ma_keys; res = PyDict_SetItem(dict, key, value); - if (cached != ((PyDictObject *)dict)->ma_keys) { - /* Either update tp->ht_cached_keys or delete it */ + if (was_shared && cached != ((PyDictObject *)dict)->ma_keys) { + /* PyDict_SetItem() may call dictresize and convert split table + * into combined table. In such case, convert it to split + * table again and update type's shared key only when this is + * the only dict sharing key with the type. + * + * This is to allow using shared key in class like this: + * + * class C: + * def __init__(self): + * # one dict resize happens + * self.a, self.b, self.c = 1, 2, 3 + * self.d, self.e, self.f = 4, 5, 6 + * a = C() + */ if (cached->dk_refcnt == 1) { CACHED_KEYS(tp) = make_keys_shared(dict); } From 4f9e4285d2c02dc0d7e0984405b04e808dd4fc05 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 15 Dec 2016 17:36:05 -0500 Subject: [PATCH 09/14] Issue #28091: Document PEP 525 & PEP 530. Patch by Eric Appelt. (grafted from 78c8f450b84ca1864123ec487d363eb151f61a4a) --- Doc/glossary.rst | 28 +++++ Doc/library/asyncio-eventloop.rst | 18 ++++ Doc/library/inspect.rst | 21 ++++ Doc/library/sys.rst | 36 +++++++ Doc/library/types.rst | 8 ++ Doc/reference/compound_stmts.rst | 2 +- Doc/reference/datamodel.rst | 19 ++++ Doc/reference/expressions.rst | 166 +++++++++++++++++++++++++++++- Doc/reference/simple_stmts.rst | 4 + 9 files changed, 298 insertions(+), 4 deletions(-) diff --git a/Doc/glossary.rst b/Doc/glossary.rst index f80b6df2126..41ee3d83b31 100644 --- a/Doc/glossary.rst +++ b/Doc/glossary.rst @@ -74,6 +74,34 @@ Glossary :keyword:`async with` statement by defining :meth:`__aenter__` and :meth:`__aexit__` methods. Introduced by :pep:`492`. + asynchronous generator + A function which returns an :term:`asynchronous generator iterator`. It + looks like a coroutine function defined with :keyword:`async def` except + that it contains :keyword:`yield` expressions for producing a series of + values usable in an :keyword:`async for` loop. + + Usually refers to a asynchronous generator function, but may refer to an + *asynchronous generator iterator* in some contexts. In cases where the + intended meaning isn't clear, using the full terms avoids ambiguity. + + An asynchronous generator function may contain :keyword:`await` + expressions as well as :keyword:`async for`, and :keyword:`async with` + statements. + + asynchronous generator iterator + An object created by a :term:`asynchronous generator` function. + + This is an :term:`asynchronous iterator` which when called using the + :meth:`__anext__` method returns an awaitable object which will execute + that the body of the asynchronous generator function until the + next :keyword:`yield` expression. + + Each :keyword:`yield` temporarily suspends processing, remembering the + location execution state (including local variables and pending + try-statements). When the *asynchronous generator iterator* effectively + resumes with another awaitable returned by :meth:`__anext__`, it + picks-up where it left-off. See :pep:`492` and :pep:`525`. + asynchronous iterable An object, that can be used in an :keyword:`async for` statement. Must return an :term:`asynchronous iterator` from its diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst index bb602c663e9..fa6a29604ca 100644 --- a/Doc/library/asyncio-eventloop.rst +++ b/Doc/library/asyncio-eventloop.rst @@ -88,6 +88,24 @@ Run an event loop This is idempotent and irreversible. No other methods should be called after this one. + +.. coroutinemethod:: AbstractEventLoop.shutdown_asyncgens() + + Schedule all currently open :term:`asynchronous generator` objects to + close with an :meth:`~agen.aclose()` call. After calling this method, + the event loop will issue a warning whenever a new asynchronous generator + is iterated. Should be used to finalize all scheduled asynchronous + generators reliably. Example:: + + try: + loop.run_forever() + finally: + loop.run_until_complete(loop.shutdown_asyncgens()) + loop.close() + + .. versionadded:: 3.6 + + .. _asyncio-pass-keywords: Calls diff --git a/Doc/library/inspect.rst b/Doc/library/inspect.rst index de0c301c1e4..41a784d982d 100644 --- a/Doc/library/inspect.rst +++ b/Doc/library/inspect.rst @@ -318,6 +318,27 @@ attributes: .. versionadded:: 3.5 +.. function:: isasyncgenfunction(object) + + Return true if the object is an :term:`asynchronous generator` function, + for example:: + + >>> async def agen(): + ... yield 1 + ... + >>> inspect.isasyncgenfunction(agen) + True + + .. versionadded:: 3.6 + + +.. function:: isasyncgen(object) + + Return true if the object is an :term:`asynchronous generator iterator` + created by an :term:`asynchronous generator` function. + + .. versionadded:: 3.6 + .. function:: istraceback(object) Return true if the object is a traceback. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index 2d14a1dd18d..dd51ffd56cc 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -594,6 +594,24 @@ always available. .. versionchanged:: 3.6 Added *platform_version* + +.. function:: get_asyncgen_hooks() + + Returns an *asyncgen_hooks* object, which is similar to a + :class:`~collections.namedtuple` of the form `(firstiter, finalizer)`, + where *firstiter* and *finalizer* are expected to be either ``None`` or + functions which take an :term:`asynchronous generator iterator` as an + argument, and are used to schedule finalization of an asychronous + generator by an event loop. + + .. versionadded:: 3.6 + See :pep:`525` for more details. + + .. note:: + This function has been added on a provisional basis (see :pep:`411` + for details.) + + .. function:: get_coroutine_wrapper() Returns ``None``, or a wrapper set by :func:`set_coroutine_wrapper`. @@ -1098,6 +1116,24 @@ always available. implementation platform, rather than part of the language definition, and thus may not be available in all Python implementations. +.. function:: set_asyncgen_hooks(firstiter, finalizer) + + Accepts two optional keyword arguments which are callables that accept an + :term:`asynchronous generator iterator` as an argument. The *firstiter* + callable will be called when an asynchronous generator is iterated for the + first time. The *finalizer* will be called when an asynchronous generator + is about to be garbage collected. + + .. versionadded:: 3.6 + See :pep:`525` for more details, and for a reference example of a + *finalizer* method see the implementation of + ``asyncio.Loop.shutdown_asyncgens`` in + :source:`Lib/asyncio/base_events.py` + + .. note:: + This function has been added on a provisional basis (see :pep:`411` + for details.) + .. function:: set_coroutine_wrapper(wrapper) diff --git a/Doc/library/types.rst b/Doc/library/types.rst index 898b95a940c..0c5619c713e 100644 --- a/Doc/library/types.rst +++ b/Doc/library/types.rst @@ -104,6 +104,14 @@ Standard names are defined for the following types: .. versionadded:: 3.5 +.. data:: AsyncGeneratorType + + The type of :term:`asynchronous generator`-iterator objects, created by + asynchronous generator functions. + + .. versionadded:: 3.6 + + .. data:: CodeType .. index:: builtin: compile diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst index 4fc6af02f12..4b425a48209 100644 --- a/Doc/reference/compound_stmts.rst +++ b/Doc/reference/compound_stmts.rst @@ -697,7 +697,7 @@ coroutine bodies. Functions defined with ``async def`` syntax are always coroutine functions, even if they do not contain ``await`` or ``async`` keywords. -It is a :exc:`SyntaxError` to use :keyword:`yield` expressions in +It is a :exc:`SyntaxError` to use ``yield from`` expressions in ``async def`` coroutines. An example of a coroutine function:: diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst index 36a9037d165..82e35e5cd1a 100644 --- a/Doc/reference/datamodel.rst +++ b/Doc/reference/datamodel.rst @@ -627,6 +627,25 @@ Callable types as well as :keyword:`async with` and :keyword:`async for` statements. See also the :ref:`coroutine-objects` section. + Asynchronous generator functions + .. index:: + single: asynchronous generator; function + single: asynchronous generator; asynchronous iterator + + A function or method which is defined using :keyword:`async def` and + which uses the :keyword:`yield` statement is called a + :dfn:`asynchronous generator function`. Such a function, when called, + returns an asynchronous iterator object which can be used in an + :keyword:`async for` statement to execute the body of the function. + + Calling the asynchronous iterator's :meth:`aiterator.__anext__` method + will return an :term:`awaitable` which when awaited + will execute until it provides a value using the :keyword:`yield` + expression. When the function executes an empty :keyword:`return` + statement or falls off the end, a :exc:`StopAsyncIteration` exception + is raised and the asynchronous iterator will have reached the end of + the set of values to be yielded. + Built-in functions .. index:: object: built-in function diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst index 08938b23c23..39c33bc5666 100644 --- a/Doc/reference/expressions.rst +++ b/Doc/reference/expressions.rst @@ -172,7 +172,7 @@ Common syntax elements for comprehensions are: .. productionlist:: comprehension: `expression` `comp_for` - comp_for: "for" `target_list` "in" `or_test` [`comp_iter`] + comp_for: [ASYNC] "for" `target_list` "in" `or_test` [`comp_iter`] comp_iter: `comp_for` | `comp_if` comp_if: "if" `expression_nocond` [`comp_iter`] @@ -186,6 +186,17 @@ each time the innermost block is reached. Note that the comprehension is executed in a separate scope, so names assigned to in the target list don't "leak" into the enclosing scope. +Since Python 3.6, in an :keyword:`async def` function, an :keyword:`async for` +clause may be used to iterate over a :term:`asynchronous iterator`. +A comprehension in an :keyword:`async def` function may consist of either a +:keyword:`for` or :keyword:`async for` clause following the leading +expression, may contan additonal :keyword:`for` or :keyword:`async for` +clauses, and may also use :keyword:`await` expressions. +If a comprehension contains either :keyword:`async for` clauses +or :keyword:`await` expressions it is called an +:dfn:`asynchronous comprehension`. An asynchronous comprehension may +suspend the execution of the coroutine function in which it appears. +See also :pep:`530`. .. _lists: @@ -315,6 +326,14 @@ range(10) for y in bar(x))``. The parentheses can be omitted on calls with only one argument. See section :ref:`calls` for details. +Since Python 3.6, if the generator appears in an :keyword:`async def` function, +then :keyword:`async for` clauses and :keyword:`await` expressions are permitted +as with an asynchronous comprehension. If a generator expression +contains either :keyword:`async for` clauses or :keyword:`await` expressions +it is called an :dfn:`asynchronous generator expression`. +An asynchronous generator expression yields a new asynchronous +generator object, which is an asynchronous iterator +(see :ref:`async-iterators`). .. _yieldexpr: @@ -330,9 +349,22 @@ Yield expressions yield_atom: "(" `yield_expression` ")" yield_expression: "yield" [`expression_list` | "from" `expression`] -The yield expression is only used when defining a :term:`generator` function and +The yield expression is used when defining a :term:`generator` function +or an :term:`asynchronous generator` function and thus can only be used in the body of a function definition. Using a yield -expression in a function's body causes that function to be a generator. +expression in a function's body causes that function to be a generator, +and using it in an :keyword:`async def` function's body causes that +coroutine function to be an asynchronous generator. For example:: + + def gen(): # defines a generator function + yield 123 + + async def agen(): # defines an asynchronous generator function (PEP 525) + yield 123 + +Generator functions are described below, while asynchronous generator +functions are described separately in section +:ref:`asynchronous-generator-functions`. When a generator function is called, it returns an iterator known as a generator. That generator then controls the execution of the generator function. @@ -496,6 +528,134 @@ generator functions:: For examples using ``yield from``, see :ref:`pep-380` in "What's New in Python." +.. _asynchronous-generator-functions: + +Asynchronous generator functions +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The presence of a yield expression in a function or method defined using +:keyword:`async def` further defines the function as a +:term:`asynchronous generator` function. + +When an asynchronous generator function is called, it returns an +asynchronous iterator known as an asynchronous generator object. +That object then controls the execution of the generator function. +An asynchronous generator object is typically used in an +:keyword:`async for` statement in a coroutine function analogously to +how a generator object would be used in a :keyword:`for` statement. + +Calling one of the asynchronous generator's methods returns an +:term:`awaitable` object, and the execution starts when this object +is awaited on. At that time, the execution proceeds to the first yield +expression, where it is suspended again, returning the value of +:token:`expression_list` to the awaiting coroutine. As with a generator, +suspension means that all local state is retained, including the +current bindings of local variables, the instruction pointer, the internal +evaluation stack, and the state of any exception handling. When the execution +is resumed by awaiting on the next object returned by the asynchronous +generator's methods, the function can proceed exactly as if the yield +expression were just another external call. The value of the yield expression +after resuming depends on the method which resumed the execution. If +:meth:`~agen.__anext__` is used then the result is :const:`None`. Otherwise, if +:meth:`~agen.asend` is used, then the result will be the value passed in to +that method. + +In an asynchronous generator function, yield expressions are allowed anywhere +in a :keyword:`try` construct. However, if an asynchronous generator is not +resumed before it is finalized (by reaching a zero reference count or by +being garbage collected), then a yield expression within a :keyword:`try` +construct could result in a failure to execute pending :keyword:`finally` +clauses. In this case, it is the responsibility of the event loop or +scheduler running the asynchronous generator to call the asynchronous +generator-iterator's :meth:`~agen.aclose` method and run the resulting +coroutine object, thus allowing any pending :keyword:`finally` clauses +to execute. + +To take care of finalization, an event loop should define +a *finalizer* function which takes an asynchronous generator-iterator +and presumably calls :meth:`~agen.aclose` and executes the coroutine. +This *finalizer* may be registered by calling :func:`sys.set_asyncgen_hooks`. +When first iterated over, an asynchronous generator-iterator will store the +registered *finalizer* to be called upon finalization. For a reference example +of a *finalizer* method see the implementation of +``asyncio.Loop.shutdown_asyncgens`` in :source:`Lib/asyncio/base_events.py`. + +The expression ``yield from `` is a syntax error when used in an +asynchronous generator function. + +.. index:: object: asynchronous-generator +.. _asynchronous-generator-methods: + +Asynchronous generator-iterator methods +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This subsection describes the methods of an asynchronous generator iterator, +which are used to control the execution of a generator function. + + +.. index:: exception: StopAsyncIteration + +.. coroutinemethod:: agen.__anext__() + + Returns an awaitable which when run starts to execute the asynchronous + generator or resumes it at the last executed yield expression. When an + asynchronous generator function is resumed with a :meth:`~agen.__anext__` + method, the current yield expression always evaluates to :const:`None` in + the returned awaitable, which when run will continue to the next yield + expression. The value of the :token:`expression_list` of the yield + expression is the value of the :exc:`StopIteration` exception raised by + the completing coroutine. If the asynchronous generator exits without + yielding another value, the awaitable instead raises an + :exc:`StopAsyncIteration` exception, signalling that the asynchronous + iteration has completed. + + This method is normally called implicitly by a :keyword:`async for` loop. + + +.. coroutinemethod:: agen.asend(value) + + Returns an awaitable which when run resumes the execution of the + asynchronous generator. As with the :meth:`~generator.send()` method for a + generator, this "sends" a value into the asynchronous generator function, + and the *value* argument becomes the result of the current yield expression. + The awaitable returned by the :meth:`asend` method will return the next + value yielded by the generator as the value of the raised + :exc:`StopIteration`, or raises :exc:`StopAsyncIteration` if the + asynchronous generator exits without yielding another value. When + :meth:`asend` is called to start the asynchronous + generator, it must be called with :const:`None` as the argument, + because there is no yield expression that could receive the value. + + +.. coroutinemethod:: agen.athrow(type[, value[, traceback]]) + + Returns an awaitable that raises an exception of type ``type`` at the point + where the asynchronous generator was paused, and returns the next value + yielded by the generator function as the value of the raised + :exc:`StopIteration` exception. If the asynchronous generator exits + without yielding another value, an :exc:`StopAsyncIteration` exception is + raised by the awaitable. + If the generator function does not catch the passed-in exception, or + raises a different exception, then when the awaitalbe is run that exception + propagates to the caller of the awaitable. + +.. index:: exception: GeneratorExit + + +.. coroutinemethod:: agen.aclose() + + Returns an awaitable that when run will throw a :exc:`GeneratorExit` into + the asynchronous generator function at the point where it was paused. + If the asynchronous generator function then exits gracefully, is already + closed, or raises :exc:`GeneratorExit` (by not catching the exception), + then the returned awaitable will raise a :exc:`StopIteration` exception. + Any further awaitables returned by subsequent calls to the asynchronous + generator will raise a :exc:`StopAsyncIteration` exception. If the + asynchronous generator yields a value, a :exc:`RuntimeError` is raised + by the awaitable. If the asynchronous generator raises any other exception, + it is propagated to the caller of the awaitable. If the asynchronous + generator has already exited due to an exception or normal exit, then + further calls to :meth:`aclose` will return an awaitable that does nothing. .. _primaries: diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst index 3dc4418f97e..e152b16ee32 100644 --- a/Doc/reference/simple_stmts.rst +++ b/Doc/reference/simple_stmts.rst @@ -492,6 +492,10 @@ generator is done and will cause :exc:`StopIteration` to be raised. The returned value (if any) is used as an argument to construct :exc:`StopIteration` and becomes the :attr:`StopIteration.value` attribute. +In an asynchronous generator function, an empty :keyword:`return` statement +indicates that the asynchronous generator is done and will cause +:exc:`StopAsyncIteration` to be raised. A non-empty :keyword:`return` +statement is a syntax error in an asynchronous generator function. .. _yield: From 6e3123590210c37e488fbe07f64be5d0748b88b8 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 15 Dec 2016 17:56:43 -0500 Subject: [PATCH 10/14] Issue #28635: asyncio-related fixes and additions. (grafted from 418ba3a0f090ac0e17a935b7cd5a63ea8263a914) --- Doc/whatsnew/3.6.rst | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst index 84c452a0493..922ee646191 100644 --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -859,7 +859,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 function if the address is already resolved. (Contributed by A. Jesse Jiryu Davis.) -* The :meth:`BaseEventLoop.stop() ` +* The :meth:`loop.stop() ` method has been changed to stop the loop immediately after the current iteration. Any new callbacks scheduled as a result of the last iteration will be discarded. @@ -870,7 +870,7 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 the :exc:`StopIteration` exception. (Contributed by Chris Angelico in :issue:`26221`.) -* New :meth:`Loop.connect_accepted_socket() ` +* New :meth:`loop.connect_accepted_socket() ` method to be used by servers that accept connections outside of asyncio, but that use asyncio to handle them. (Contributed by Jim Fulton in :issue:`27392`.) @@ -878,6 +878,17 @@ Notable changes in the :mod:`asyncio` module since Python 3.5.0 * ``TCP_NODELAY`` flag is now set for all TCP transports by default. (Contributed by Yury Selivanov in :issue:`27456`.) +* New :meth:`loop.shutdown_asyncgens() ` + to properly close pending asynchronous generators before closing the + loop. + (Contributed by Yury Selivanov in :issue:`28003`.) + +* :class:`Future ` and :class:`Task ` + classes now have an optimized C implementation which makes asyncio + code up to 30% faster. + (Contributed by Yury Selivanov and INADA Naoki in :issue:`26081` + and :issue:`28544`.) + binascii -------- @@ -1714,11 +1725,10 @@ Optimizations (Contributed by Demur Rumed with input and reviews from Serhiy Storchaka and Victor Stinner in :issue:`26647` and :issue:`28050`.) -* The :class:`Future ` class now has an optimized - C implementation. - (Contributed by Yury Selivanov and INADA Naoki in :issue:`26801`.) +* The :class:`asyncio.Future` class now has an optimized C implementation. + (Contributed by Yury Selivanov and INADA Naoki in :issue:`26081`.) -* The :class:`Task ` class now has an optimized +* The :class:`asyncio.Task` class now has an optimized C implementation. (Contributed by Yury Selivanov in :issue:`28544`.) * Various implementation improvements in the :mod:`typing` module From 8e1baa7583df3594d056197638c5b12c109eae11 Mon Sep 17 00:00:00 2001 From: Yury Selivanov Date: Thu, 15 Dec 2016 18:58:19 -0500 Subject: [PATCH 11/14] docs: asyncio is no longer provisional (grafted from 4cb3ea76ce68efd52271e499647abbf0f8a2941f) --- Doc/library/asyncio.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst index 76bd9e98482..b076b7d0093 100644 --- a/Doc/library/asyncio.rst +++ b/Doc/library/asyncio.rst @@ -8,13 +8,6 @@ **Source code:** :source:`Lib/asyncio/` -.. note:: - - The asyncio package has been included in the standard library on a - :term:`provisional basis `. Backwards incompatible - changes (up to and including removal of the module) may occur if deemed - necessary by the core developers. - -------------- This module provides infrastructure for writing single-threaded concurrent From daaaff9277aea5b7fa72b9f2dd26612538e0b640 Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Thu, 15 Dec 2016 23:20:48 -0500 Subject: [PATCH 12/14] Issue #28898: add Misc/NEWS entry --- Misc/NEWS | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Misc/NEWS b/Misc/NEWS index 3e1517939a4..5cf929f21f8 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -19,6 +19,11 @@ Windows - Issue #28896: Deprecate WindowsRegistryFinder +Build +----- + +- Issue #28898: Prevent gdb build errors due to HAVE_LONG_LONG redefinition. + What's New in Python 3.6.0 release candidate 1 ============================================== From 86368b8529f784bd7a70f280931e835c07a8739c Mon Sep 17 00:00:00 2001 From: Ned Deily Date: Fri, 16 Dec 2016 00:13:46 -0500 Subject: [PATCH 13/14] bump version to 3.6.0rc1+ --- Include/patchlevel.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Include/patchlevel.h b/Include/patchlevel.h index e2a9aceb98b..f59e12ada13 100644 --- a/Include/patchlevel.h +++ b/Include/patchlevel.h @@ -23,7 +23,7 @@ #define PY_RELEASE_SERIAL 1 /* Version as a string */ -#define PY_VERSION "3.6.0rc1" +#define PY_VERSION "3.6.0rc1+" /*--end constants--*/ /* Version as a single 4-byte hex number, e.g. 0x010502B2 == 1.5.2b2. From 610f5d739dd22bce352bde59dce3985c73aaefab Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 16 Dec 2016 10:00:39 +0100 Subject: [PATCH 14/14] python-gdb.py: catch gdb.error on gdb.selected_frame() --- Tools/gdb/libpython.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py index 75f1ccbd442..ed515c00173 100755 --- a/Tools/gdb/libpython.py +++ b/Tools/gdb/libpython.py @@ -1527,7 +1527,11 @@ class Frame(object): def get_selected_python_frame(cls): '''Try to obtain the Frame for the python-related code in the selected frame, or None''' - frame = cls.get_selected_frame() + try: + frame = cls.get_selected_frame() + except gdb.error: + # No frame: Python didn't start yet + return None while frame: if frame.is_python_frame(): @@ -1668,6 +1672,10 @@ PyList() def move_in_stack(move_up): '''Move up or down the stack (for the py-up/py-down command)''' frame = Frame.get_selected_python_frame() + if not frame: + print('Unable to locate python frame') + return + while frame: if move_up: iter_frame = frame.older() @@ -1730,6 +1738,10 @@ class PyBacktraceFull(gdb.Command): def invoke(self, args, from_tty): frame = Frame.get_selected_python_frame() + if not frame: + print('Unable to locate python frame') + return + while frame: if frame.is_python_frame(): frame.print_summary() @@ -1747,8 +1759,12 @@ class PyBacktrace(gdb.Command): def invoke(self, args, from_tty): - sys.stdout.write('Traceback (most recent call first):\n') frame = Frame.get_selected_python_frame() + if not frame: + print('Unable to locate python frame') + return + + sys.stdout.write('Traceback (most recent call first):\n') while frame: if frame.is_python_frame(): frame.print_traceback()