From f74ef458ab1f502e4e60bd1502ac1dc0d2cb3847 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 15 Dec 2017 07:04:38 +0200 Subject: [PATCH] bpo-32311: Implement asyncio.create_task() shortcut (#4848) * Implement functionality * Add documentation --- Doc/library/asyncio-task.rst | 25 +++++- Lib/asyncio/base_futures.py | 4 +- Lib/asyncio/constants.py | 2 +- Lib/asyncio/coroutines.py | 10 +-- Lib/asyncio/events.py | 83 ++----------------- Lib/asyncio/format_helpers.py | 75 +++++++++++++++++ Lib/asyncio/futures.py | 4 +- Lib/asyncio/tasks.py | 23 ++++- Lib/test/test_asyncio/test_tasks.py | 37 +++++++++ Lib/test/test_asyncio/utils.py | 3 +- .../2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst | 1 + Modules/_asynciomodule.c | 29 ++++++- 12 files changed, 201 insertions(+), 95 deletions(-) create mode 100644 Lib/asyncio/format_helpers.py create mode 100644 Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst index 0d0569f0ba1..72fae5e8559 100644 --- a/Doc/library/asyncio-task.rst +++ b/Doc/library/asyncio-task.rst @@ -371,10 +371,21 @@ with the result. Task ---- +.. function:: create_task(coro) + + Wrap a :ref:`coroutine ` *coro* into a task and schedule + its execution. Return the task object. + + The task is executed in :func:`get_running_loop` context, + :exc:`RuntimeError` is raised if there is no running loop in + current thread. + + .. versionadded:: 3.7 + .. class:: Task(coro, \*, loop=None) - Schedule the execution of a :ref:`coroutine `: wrap it in a - future. A task is a subclass of :class:`Future`. + A unit for concurrent running of :ref:`coroutines `, + subclass of :class:`Future`. A task is responsible for executing a coroutine object in an event loop. If the wrapped coroutine yields from a future, the task suspends the execution @@ -399,7 +410,7 @@ Task ` did not complete. It is probably a bug and a warning is logged: see :ref:`Pending task destroyed `. - Don't directly create :class:`Task` instances: use the :func:`ensure_future` + Don't directly create :class:`Task` instances: use the :func:`create_task` function or the :meth:`AbstractEventLoop.create_task` method. This class is :ref:`not thread safe `. @@ -547,9 +558,15 @@ Task functions .. versionchanged:: 3.5.1 The function accepts any :term:`awaitable` object. + .. note:: + + :func:`create_task` (added in Python 3.7) is the preferable way + for spawning new tasks. + .. seealso:: - The :meth:`AbstractEventLoop.create_task` method. + The :func:`create_task` function and + :meth:`AbstractEventLoop.create_task` method. .. function:: wrap_future(future, \*, loop=None) diff --git a/Lib/asyncio/base_futures.py b/Lib/asyncio/base_futures.py index 2ee82c3f057..008812eda91 100644 --- a/Lib/asyncio/base_futures.py +++ b/Lib/asyncio/base_futures.py @@ -3,7 +3,7 @@ __all__ = () import concurrent.futures._base import reprlib -from . import events +from . import format_helpers Error = concurrent.futures._base.Error CancelledError = concurrent.futures.CancelledError @@ -38,7 +38,7 @@ def _format_callbacks(cb): cb = '' def format_cb(callback): - return events._format_callback_source(callback, ()) + return format_helpers._format_callback_source(callback, ()) if size == 1: cb = format_cb(cb[0]) diff --git a/Lib/asyncio/constants.py b/Lib/asyncio/constants.py index dfe97f49859..52169c3f8e5 100644 --- a/Lib/asyncio/constants.py +++ b/Lib/asyncio/constants.py @@ -6,5 +6,5 @@ ACCEPT_RETRY_DELAY = 1 # Number of stack entries to capture in debug mode. # The larger the number, the slower the operation in debug mode -# (see extract_stack() in events.py). +# (see extract_stack() in format_helpers.py). DEBUG_STACK_DEPTH = 10 diff --git a/Lib/asyncio/coroutines.py b/Lib/asyncio/coroutines.py index bca7fe3a537..e3c0162dd14 100644 --- a/Lib/asyncio/coroutines.py +++ b/Lib/asyncio/coroutines.py @@ -9,9 +9,9 @@ import types from collections.abc import Awaitable, Coroutine -from . import constants -from . import events from . import base_futures +from . import constants +from . import format_helpers from .log import logger @@ -48,7 +48,7 @@ class CoroWrapper: assert inspect.isgenerator(gen) or inspect.iscoroutine(gen), gen self.gen = gen self.func = func # Used to unwrap @coroutine decorator - self._source_traceback = events.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack(sys._getframe(1)) self.__name__ = getattr(gen, '__name__', None) self.__qualname__ = getattr(gen, '__qualname__', None) @@ -243,7 +243,7 @@ def _format_coroutine(coro): func = coro if coro_name is None: - coro_name = events._format_callback(func, (), {}) + coro_name = format_helpers._format_callback(func, (), {}) try: coro_code = coro.gi_code @@ -260,7 +260,7 @@ def _format_coroutine(coro): if (isinstance(coro, CoroWrapper) and not inspect.isgeneratorfunction(coro.func) and coro.func is not None): - source = events._get_function_source(coro.func) + source = format_helpers._get_function_source(coro.func) if source is not None: filename, lineno = source if coro_frame is None: diff --git a/Lib/asyncio/events.py b/Lib/asyncio/events.py index a00f861a9e5..974a4a22218 100644 --- a/Lib/asyncio/events.py +++ b/Lib/asyncio/events.py @@ -11,86 +11,14 @@ __all__ = ( '_get_running_loop', ) -import functools -import inspect import os -import reprlib import socket import subprocess import sys import threading -import traceback from . import constants - - -def _get_function_source(func): - func = inspect.unwrap(func) - if inspect.isfunction(func): - code = func.__code__ - return (code.co_filename, code.co_firstlineno) - if isinstance(func, functools.partial): - return _get_function_source(func.func) - if isinstance(func, functools.partialmethod): - return _get_function_source(func.func) - return None - - -def _format_args_and_kwargs(args, kwargs): - """Format function arguments and keyword arguments. - - Special case for a single parameter: ('hello',) is formatted as ('hello'). - """ - # use reprlib to limit the length of the output - items = [] - if args: - items.extend(reprlib.repr(arg) for arg in args) - if kwargs: - items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items()) - return '({})'.format(', '.join(items)) - - -def _format_callback(func, args, kwargs, suffix=''): - if isinstance(func, functools.partial): - suffix = _format_args_and_kwargs(args, kwargs) + suffix - return _format_callback(func.func, func.args, func.keywords, suffix) - - if hasattr(func, '__qualname__'): - func_repr = getattr(func, '__qualname__') - elif hasattr(func, '__name__'): - func_repr = getattr(func, '__name__') - else: - func_repr = repr(func) - - func_repr += _format_args_and_kwargs(args, kwargs) - if suffix: - func_repr += suffix - return func_repr - - -def _format_callback_source(func, args): - func_repr = _format_callback(func, args, None) - source = _get_function_source(func) - if source: - func_repr += f' at {source[0]}:{source[1]}' - return func_repr - - -def extract_stack(f=None, limit=None): - """Replacement for traceback.extract_stack() that only does the - necessary work for asyncio debug mode. - """ - if f is None: - f = sys._getframe().f_back - if limit is None: - # Limit the amount of work to a reasonable amount, as extract_stack() - # can be called for each coroutine and future in debug mode. - limit = constants.DEBUG_STACK_DEPTH - stack = traceback.StackSummary.extract(traceback.walk_stack(f), - limit=limit, - lookup_lines=False) - stack.reverse() - return stack +from . import format_helpers class Handle: @@ -106,7 +34,8 @@ class Handle: self._cancelled = False self._repr = None if self._loop.get_debug(): - self._source_traceback = extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) else: self._source_traceback = None @@ -115,7 +44,8 @@ class Handle: if self._cancelled: info.append('cancelled') if self._callback is not None: - info.append(_format_callback_source(self._callback, self._args)) + info.append(format_helpers._format_callback_source( + self._callback, self._args)) if self._source_traceback: frame = self._source_traceback[-1] info.append(f'created at {frame[0]}:{frame[1]}') @@ -145,7 +75,8 @@ class Handle: try: self._callback(*self._args) except Exception as exc: - cb = _format_callback_source(self._callback, self._args) + cb = format_helpers._format_callback_source( + self._callback, self._args) msg = f'Exception in callback {cb}' context = { 'message': msg, diff --git a/Lib/asyncio/format_helpers.py b/Lib/asyncio/format_helpers.py new file mode 100644 index 00000000000..39cfcee0c1c --- /dev/null +++ b/Lib/asyncio/format_helpers.py @@ -0,0 +1,75 @@ +import functools +import inspect +import reprlib +import traceback + +from . import constants + + +def _get_function_source(func): + func = inspect.unwrap(func) + if inspect.isfunction(func): + code = func.__code__ + return (code.co_filename, code.co_firstlineno) + if isinstance(func, functools.partial): + return _get_function_source(func.func) + if isinstance(func, functools.partialmethod): + return _get_function_source(func.func) + return None + + +def _format_callback_source(func, args): + func_repr = _format_callback(func, args, None) + source = _get_function_source(func) + if source: + func_repr += f' at {source[0]}:{source[1]}' + return func_repr + + +def _format_args_and_kwargs(args, kwargs): + """Format function arguments and keyword arguments. + + Special case for a single parameter: ('hello',) is formatted as ('hello'). + """ + # use reprlib to limit the length of the output + items = [] + if args: + items.extend(reprlib.repr(arg) for arg in args) + if kwargs: + items.extend(f'{k}={reprlib.repr(v)}' for k, v in kwargs.items()) + return '({})'.format(', '.join(items)) + + +def _format_callback(func, args, kwargs, suffix=''): + if isinstance(func, functools.partial): + suffix = _format_args_and_kwargs(args, kwargs) + suffix + return _format_callback(func.func, func.args, func.keywords, suffix) + + if hasattr(func, '__qualname__'): + func_repr = getattr(func, '__qualname__') + elif hasattr(func, '__name__'): + func_repr = getattr(func, '__name__') + else: + func_repr = repr(func) + + func_repr += _format_args_and_kwargs(args, kwargs) + if suffix: + func_repr += suffix + return func_repr + + +def extract_stack(f=None, limit=None): + """Replacement for traceback.extract_stack() that only does the + necessary work for asyncio debug mode. + """ + if f is None: + f = sys._getframe().f_back + if limit is None: + # Limit the amount of work to a reasonable amount, as extract_stack() + # can be called for each coroutine and future in debug mode. + limit = constants.DEBUG_STACK_DEPTH + stack = traceback.StackSummary.extract(traceback.walk_stack(f), + limit=limit, + lookup_lines=False) + stack.reverse() + return stack diff --git a/Lib/asyncio/futures.py b/Lib/asyncio/futures.py index d46a295197b..b310962f9fe 100644 --- a/Lib/asyncio/futures.py +++ b/Lib/asyncio/futures.py @@ -11,6 +11,7 @@ import sys from . import base_futures from . import events +from . import format_helpers CancelledError = base_futures.CancelledError @@ -79,7 +80,8 @@ class Future: self._loop = loop self._callbacks = [] if self._loop.get_debug(): - self._source_traceback = events.extract_stack(sys._getframe(1)) + self._source_traceback = format_helpers.extract_stack( + sys._getframe(1)) _repr_info = base_futures._future_repr_info diff --git a/Lib/asyncio/tasks.py b/Lib/asyncio/tasks.py index c5122f76071..172057e5a29 100644 --- a/Lib/asyncio/tasks.py +++ b/Lib/asyncio/tasks.py @@ -1,7 +1,7 @@ """Support for tasks, coroutines and the scheduler.""" __all__ = ( - 'Task', + 'Task', 'create_task', 'FIRST_COMPLETED', 'FIRST_EXCEPTION', 'ALL_COMPLETED', 'wait', 'wait_for', 'as_completed', 'sleep', 'gather', 'shield', 'ensure_future', 'run_coroutine_threadsafe', @@ -67,13 +67,19 @@ class Task(futures.Future): return {t for t in cls._all_tasks if t._loop is loop} def __init__(self, coro, *, loop=None): - assert coroutines.iscoroutine(coro), repr(coro) super().__init__(loop=loop) if self._source_traceback: del self._source_traceback[-1] - self._coro = coro - self._fut_waiter = None + if not coroutines.iscoroutine(coro): + # raise after Future.__init__(), attrs are required for __del__ + # prevent logging for pending task in __del__ + self._log_destroy_pending = False + raise TypeError(f"a coroutine was expected, got {coro!r}") + self._must_cancel = False + self._fut_waiter = None + self._coro = coro + self._loop.call_soon(self._step) self.__class__._all_tasks.add(self) @@ -263,6 +269,15 @@ else: Task = _CTask = _asyncio.Task +def create_task(coro): + """Schedule the execution of a coroutine object in a spawn task. + + Return a Task object. + """ + loop = events.get_running_loop() + return loop.create_task(coro) + + # wait() and as_completed() similar to those in PEP 3148. FIRST_COMPLETED = concurrent.futures.FIRST_COMPLETED diff --git a/Lib/test/test_asyncio/test_tasks.py b/Lib/test/test_asyncio/test_tasks.py index a5563ba9c6c..a32dca13118 100644 --- a/Lib/test/test_asyncio/test_tasks.py +++ b/Lib/test/test_asyncio/test_tasks.py @@ -2054,6 +2054,43 @@ class BaseTaskTests: self.assertEqual(self.Task.all_tasks(self.loop), set()) + def test_create_task_with_noncoroutine(self): + with self.assertRaisesRegex(TypeError, + "a coroutine was expected, got 123"): + self.new_task(self.loop, 123) + + def test_create_task_with_oldstyle_coroutine(self): + + @asyncio.coroutine + def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_create_task_with_async_function(self): + + async def coro(): + pass + + task = self.new_task(self.loop, coro()) + self.assertIsInstance(task, self.Task) + self.loop.run_until_complete(task) + + def test_bare_create_task(self): + + async def inner(): + return 1 + + async def coro(): + task = asyncio.create_task(inner()) + self.assertIsInstance(task, self.Task) + ret = await task + self.assertEqual(1, ret) + + self.loop.run_until_complete(coro()) + def add_subclass_tests(cls): BaseTask = cls.Task diff --git a/Lib/test/test_asyncio/utils.py b/Lib/test/test_asyncio/utils.py index 560db9f562d..a1a9bb3684c 100644 --- a/Lib/test/test_asyncio/utils.py +++ b/Lib/test/test_asyncio/utils.py @@ -28,6 +28,7 @@ except ImportError: # pragma: no cover from asyncio import base_events from asyncio import events +from asyncio import format_helpers from asyncio import futures from asyncio import tasks from asyncio.log import logger @@ -429,7 +430,7 @@ class MockPattern(str): def get_function_source(func): - source = events._get_function_source(func) + source = format_helpers._get_function_source(func) if source is None: raise ValueError("unable to get the source of %r" % (func,)) return source diff --git a/Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst b/Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst new file mode 100644 index 00000000000..e2d10959476 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2017-12-14-17-28-54.bpo-32311.DL5Ytn.rst @@ -0,0 +1 @@ +Implement asyncio.create_task(coro) shortcut diff --git a/Modules/_asynciomodule.c b/Modules/_asynciomodule.c index 01c38b80b95..9ac1c44d48d 100644 --- a/Modules/_asynciomodule.c +++ b/Modules/_asynciomodule.c @@ -26,6 +26,7 @@ static PyObject *all_tasks; static PyObject *current_tasks; static PyObject *traceback_extract_stack; static PyObject *asyncio_get_event_loop_policy; +static PyObject *asyncio_iscoroutine_func; static PyObject *asyncio_future_repr_info_func; static PyObject *asyncio_task_repr_info_func; static PyObject *asyncio_task_get_stack_func; @@ -1461,16 +1462,38 @@ _asyncio_Task___init___impl(TaskObj *self, PyObject *coro, PyObject *loop) /*[clinic end generated code: output=9f24774c2287fc2f input=8d132974b049593e]*/ { PyObject *res; + int tmp; _Py_IDENTIFIER(add); if (future_init((FutureObj*)self, loop)) { return -1; } + if (!PyCoro_CheckExact(coro)) { + // fastpath failed, perfom slow check + // raise after Future.__init__(), attrs are required for __del__ + res = PyObject_CallFunctionObjArgs(asyncio_iscoroutine_func, + coro, NULL); + if (res == NULL) { + return -1; + } + tmp = PyObject_Not(res); + Py_DECREF(res); + if (tmp < 0) { + return -1; + } + if (tmp) { + self->task_log_destroy_pending = 0; + PyErr_Format(PyExc_TypeError, + "a coroutine was expected, got %R", + coro, NULL); + return -1; + } + } + self->task_fut_waiter = NULL; self->task_must_cancel = 0; self->task_log_destroy_pending = 1; - Py_INCREF(coro); self->task_coro = coro; @@ -2604,6 +2627,7 @@ module_free(void *m) Py_CLEAR(traceback_extract_stack); Py_CLEAR(asyncio_get_event_loop_policy); Py_CLEAR(asyncio_future_repr_info_func); + Py_CLEAR(asyncio_iscoroutine_func); Py_CLEAR(asyncio_task_repr_info_func); Py_CLEAR(asyncio_task_get_stack_func); Py_CLEAR(asyncio_task_print_stack_func); @@ -2645,6 +2669,9 @@ module_init(void) GET_MOD_ATTR(asyncio_task_get_stack_func, "_task_get_stack") GET_MOD_ATTR(asyncio_task_print_stack_func, "_task_print_stack") + WITH_MOD("asyncio.coroutines") + GET_MOD_ATTR(asyncio_iscoroutine_func, "iscoroutine") + WITH_MOD("inspect") GET_MOD_ATTR(inspect_isgenerator, "isgenerator")