perform yield from delegation by repeating YIELD_FROM opcode (closes #14230)

This allows generators that are using yield from to be seen by debuggers. It
also kills the f_yieldfrom field on frame objects.

Patch mostly from Mark Shannon with a few tweaks by me.

Include/frameobject.h

@@ -27,7 +27,6 @@ typedef struct _frame {
        to the current stack top. */
     PyObject **f_stacktop;
     PyObject *f_trace;          /* Trace function */
-    PyObject *f_yieldfrom;      /* Iterator being delegated to by yield from */
 
         /* In a generator, we need to be able to swap between the exception
            state inside the generator and the exception state of the calling

Include/genobject.h

@@ -35,6 +35,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
 PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
 PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
 PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **);
+PyObject *_PyGen_Send(PyGenObject *, PyObject *);
 
 #ifdef __cplusplus
 }

Lib/test/test_pep380.py

@@ -10,7 +10,7 @@ see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Pyt
 import unittest
 import io
 import sys
-import traceback
+import inspect
 import parser
 
 from test.support import captured_stderr
@@ -919,6 +919,27 @@ class TestPEP380Operation(unittest.TestCase):
         next(g1)
         g1.close()
 
+    def test_delegator_is_visible_to_debugger(self):
+        def call_stack():
+            return [f[3] for f in inspect.stack()]
+
+        def gen():
+            yield call_stack()
+            yield call_stack()
+            yield call_stack()
+
+        def spam(g):
+            yield from g
+
+        def eggs(g):
+            yield from g
+
+        for stack in spam(gen()):
+            self.assertTrue('spam' in stack)
+
+        for stack in spam(eggs(gen())):
+            self.assertTrue('spam' in stack and 'eggs' in stack)
+
 
 def test_main():
     from test import support

Lib/test/test_sys.py

@@ -730,7 +730,7 @@ class SizeofTest(unittest.TestCase):
         nfrees = len(x.f_code.co_freevars)
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
                   ncells + nfrees - 1
-        check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+        check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
         # function
         def func(): pass
         check(func, size(h + '12P'))

Objects/frameobject.c

@@ -444,7 +444,6 @@ frame_dealloc(PyFrameObject *f)
     Py_CLEAR(f->f_exc_type);
     Py_CLEAR(f->f_exc_value);
     Py_CLEAR(f->f_exc_traceback);
-    Py_CLEAR(f->f_yieldfrom);
 
     co = f->f_code;
     if (co->co_zombieframe == NULL)
@@ -476,7 +475,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
     Py_VISIT(f->f_exc_type);
     Py_VISIT(f->f_exc_value);
     Py_VISIT(f->f_exc_traceback);
-    Py_VISIT(f->f_yieldfrom);
 
     /* locals */
     slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@@ -510,7 +508,6 @@ frame_clear(PyFrameObject *f)
     Py_CLEAR(f->f_exc_value);
     Py_CLEAR(f->f_exc_traceback);
     Py_CLEAR(f->f_trace);
-    Py_CLEAR(f->f_yieldfrom);
 
     /* locals */
     slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@@ -714,7 +711,6 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
     f->f_lasti = -1;
     f->f_lineno = code->co_firstlineno;
     f->f_iblock = 0;
-    f->f_yieldfrom = NULL;
 
     _PyObject_GC_TRACK(f);
     return f;

Objects/genobject.c

@@ -6,7 +6,6 @@
 #include "opcode.h"
 
 static PyObject *gen_close(PyGenObject *gen, PyObject *args);
-static void gen_undelegate(PyGenObject *gen);
 
 static int
 gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
@@ -41,15 +40,6 @@ gen_dealloc(PyGenObject *gen)
     PyObject_GC_Del(gen);
 }
 
-static int
-gen_running(PyGenObject *gen)
-{
-    if (gen->gi_running) {
-        PyErr_SetString(PyExc_ValueError, "generator already executing");
-        return 1;
-    }
-    return 0;
-}
 
 static PyObject *
 gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
@@ -58,7 +48,11 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
     PyFrameObject *f = gen->gi_frame;
     PyObject *result;
 
-    assert(!gen->gi_running);
+    if (gen->gi_running) {
+        PyErr_SetString(PyExc_ValueError,
+                        "generator already executing");
+        return NULL;
+    }
     if (f==NULL || f->f_stacktop == NULL) {
         /* Only set exception if called from send() */
         if (arg && !exc)
@@ -136,45 +130,10 @@ PyDoc_STRVAR(send_doc,
 "send(arg) -> send 'arg' into generator,\n\
 return next yielded value or raise StopIteration.");
 
-static PyObject *
+PyObject *
-gen_send(PyGenObject *gen, PyObject *arg)
+_PyGen_Send(PyGenObject *gen, PyObject *arg)
 {
-    int exc = 0;
+    return gen_send_ex(gen, arg, 0);
-    PyObject *ret;
-    PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
-    if (gen_running(gen))
-        return NULL;
-    /* XXX (ncoghlan): Are the incref/decref on arg and yf strictly needed?
-     *                 Or would it be valid to rely on borrowed references?
-     */
-    Py_INCREF(arg);
-    if (yf) {
-        Py_INCREF(yf);
-        gen->gi_running = 1;
-        if (PyGen_CheckExact(yf)) {
-            ret = gen_send((PyGenObject *)yf, arg);
-        } else {
-            if (arg == Py_None)
-                ret = PyIter_Next(yf);
-            else
-                ret = PyObject_CallMethod(yf, "send", "O", arg);
-        }
-        gen->gi_running = 0;
-        if (ret) {
-            Py_DECREF(yf);
-            goto done;
-        }
-        gen_undelegate(gen);
-        Py_CLEAR(arg);
-        if (PyGen_FetchStopIterationValue(&arg) < 0) {
-            exc = 1;
-        }
-        Py_DECREF(yf);
-    }
-    ret = gen_send_ex(gen, arg, exc);
-done:
-    Py_XDECREF(arg);
-    return ret;
 }
 
 PyDoc_STRVAR(close_doc,
@@ -186,49 +145,61 @@ PyDoc_STRVAR(close_doc,
  */
 
 static int
-gen_close_iter(PyGenObject *gen, PyObject *yf)
+gen_close_iter(PyObject *yf)
 {
     PyObject *retval = NULL;
-    int err = 0;
 
     if (PyGen_CheckExact(yf)) {
         retval = gen_close((PyGenObject *)yf, NULL);
-        if (!retval)
+        if (retval == NULL)
-            err = -1;
+            return -1;
     } else {
-        PyObject *meth;
+        PyObject *meth = PyObject_GetAttrString(yf, "close");
-        gen->gi_running = 1;
-        meth = PyObject_GetAttrString(yf, "close");
         if (meth == NULL) {
-            if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
+            if (!PyErr_ExceptionMatches(PyExc_AttributeError))
                 PyErr_WriteUnraisable(yf);
-            }
             PyErr_Clear();
         } else {
             retval = PyObject_CallFunction(meth, "");
             Py_DECREF(meth);
-            if (!retval)
+            if (retval == NULL)
-                err = -1;
+                return -1;
         }
-        gen->gi_running = 0;
     }
     Py_XDECREF(retval);
-    return err;
+    return 0;
+}
+
+static PyObject *
+gen_yf(PyGenObject *gen)
+{
+    PyObject *yf = NULL;
+    PyFrameObject *f = gen->gi_frame;
+
+    if (f) {
+        PyObject *bytecode = f->f_code->co_code;
+        unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
+
+        if (code[f->f_lasti + 1] != YIELD_FROM)
+            return NULL;
+        yf = f->f_stacktop[-1];
+        Py_INCREF(yf);
+    }
+
+    return yf;
 }
 
 static PyObject *
 gen_close(PyGenObject *gen, PyObject *args)
 {
     PyObject *retval;
-    PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
+    PyObject *yf = gen_yf(gen);
     int err = 0;
 
-    if (gen_running(gen))
-        return NULL;
     if (yf) {
-        Py_INCREF(yf);
+        gen->gi_running = 1;
-        err = gen_close_iter(gen, yf);
+        err = gen_close_iter(yf);
-        gen_undelegate(gen);
+        gen->gi_running = 0;
         Py_DECREF(yf);
     }
     if (err == 0)
@@ -241,8 +212,7 @@ gen_close(PyGenObject *gen, PyObject *args)
         return NULL;
     }
     if (PyErr_ExceptionMatches(PyExc_StopIteration)
-        || PyErr_ExceptionMatches(PyExc_GeneratorExit))
+        || PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
-    {
         PyErr_Clear();          /* ignore these errors */
         Py_INCREF(Py_None);
         return Py_None;
@@ -323,29 +293,27 @@ gen_throw(PyGenObject *gen, PyObject *args)
     PyObject *typ;
     PyObject *tb = NULL;
     PyObject *val = NULL;
-    PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
+    PyObject *yf = gen_yf(gen);
 
     if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))
         return NULL;
 
-    if (gen_running(gen))
-        return NULL;
-
     if (yf) {
         PyObject *ret;
         int err;
-        Py_INCREF(yf);
         if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
-            err = gen_close_iter(gen, yf);
+            gen->gi_running = 1;
+            err = gen_close_iter(yf);
+            gen->gi_running = 0;
             Py_DECREF(yf);
-            gen_undelegate(gen);
             if (err < 0)
                 return gen_send_ex(gen, Py_None, 1);
             goto throw_here;
         }
-        gen->gi_running = 1;
         if (PyGen_CheckExact(yf)) {
+            gen->gi_running = 1;
             ret = gen_throw((PyGenObject *)yf, args);
+            gen->gi_running = 0;
         } else {
             PyObject *meth = PyObject_GetAttrString(yf, "throw");
             if (meth == NULL) {
@@ -355,18 +323,22 @@ gen_throw(PyGenObject *gen, PyObject *args)
                 }
                 PyErr_Clear();
                 Py_DECREF(yf);
-                gen_undelegate(gen);
-                gen->gi_running = 0;
                 goto throw_here;
             }
+            gen->gi_running = 1;
             ret = PyObject_CallObject(meth, args);
+            gen->gi_running = 0;
             Py_DECREF(meth);
         }
-        gen->gi_running = 0;
         Py_DECREF(yf);
         if (!ret) {
             PyObject *val;
-            gen_undelegate(gen);
+            /* Pop subiterator from stack */
+            ret = *(--gen->gi_frame->f_stacktop);
+            assert(ret == yf);
+            Py_DECREF(ret);
+            /* Termination repetition of YIELD_FROM */
+            gen->gi_frame->f_lasti++;
             if (PyGen_FetchStopIterationValue(&val) == 0) {
                 ret = gen_send_ex(gen, val, 0);
                 Py_DECREF(val);
@@ -441,44 +413,11 @@ gen_iternext(PyGenObject *gen)
 {
     PyObject *val = NULL;
     PyObject *ret;
-    int exc = 0;
+    ret = gen_send_ex(gen, val, 0);
-    PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
-    if (gen_running(gen))
-        return NULL;
-    if (yf) {
-        Py_INCREF(yf);
-        /* ceval.c ensures that yf is an iterator */
-        gen->gi_running = 1;
-        ret = Py_TYPE(yf)->tp_iternext(yf);
-        gen->gi_running = 0;
-        if (ret) {
-            Py_DECREF(yf);
-            return ret;
-        }
-        gen_undelegate(gen);
-        if (PyGen_FetchStopIterationValue(&val) < 0)
-            exc = 1;
-        Py_DECREF(yf);
-    }
-    ret = gen_send_ex(gen, val, exc);
     Py_XDECREF(val);
     return ret;
 }
 
-/*
- *   In certain recursive situations, a generator may lose its frame
- *   before we get a chance to clear f_yieldfrom, so we use this
- *   helper function.
- */
-
-static void
-gen_undelegate(PyGenObject *gen) {
-    if (gen->gi_frame) {
-        Py_XDECREF(gen->gi_frame->f_yieldfrom);
-        gen->gi_frame->f_yieldfrom = NULL;
-    }
-}
-
 /*
  *   If StopIteration exception is set, fetches its 'value'
  *   attribute if any, otherwise sets pvalue to None.
@@ -548,7 +487,7 @@ static PyMemberDef gen_memberlist[] = {
 };
 
 static PyMethodDef gen_methods[] = {
-    {"send",(PyCFunction)gen_send, METH_O, send_doc},
+    {"send",(PyCFunction)_PyGen_Send, METH_O, send_doc},
     {"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc},
     {"close",(PyCFunction)gen_close, METH_NOARGS, close_doc},
     {NULL, NULL}        /* Sentinel */

Python/ceval.c

@@ -1170,6 +1170,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
        f->f_lasti to -1 (i.e. the index *before* the first instruction)
        and YIELD_VALUE doesn't fiddle with f_lasti any more.  So this
        does work.  Promise.
+       YIELD_FROM sets f_lasti to itself, in order to repeated yield
+       multiple values.
 
        When the PREDICT() macros are enabled, some opcode pairs follow in
        direct succession without updating f->f_lasti.  A successful
@@ -1830,49 +1832,35 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
 
         TARGET(YIELD_FROM)
             u = POP();
-            x = PyObject_GetIter(u);
+            x = TOP();
+            /* send u to x */
+            if (PyGen_CheckExact(x)) {
+                retval = _PyGen_Send((PyGenObject *)x, u);
+            } else {
+                if (u == Py_None)
+                    retval = PyIter_Next(x);
+                else
+                    retval = PyObject_CallMethod(x, "send", "O", u);
+            }
             Py_DECREF(u);
-            if (x == NULL)
-                break;
-            /* x is now the iterator, make the first next() call */
-            retval = (*Py_TYPE(x)->tp_iternext)(x);
             if (!retval) {
-                PyObject *et, *ev, *tb;
+                PyObject *val;
-                /* iter may be exhausted */
+                x = POP(); /* Remove iter from stack */
-                Py_CLEAR(x);
+                Py_DECREF(x);
-                if (PyErr_Occurred() &&
+                err = PyGen_FetchStopIterationValue(&val);
-                    !PyErr_ExceptionMatches(PyExc_StopIteration)) {
+                if (err < 0) {
-                    /* some other exception */
+                    x = NULL;
                     break;
                 }
-                /* try to get return value from exception */
+                x = val;
-                PyErr_Fetch(&et, &ev, &tb);
+                PUSH(x);
-                Py_XDECREF(et);
-                Py_XDECREF(tb);
-                /* u is return value */
-                u = NULL;
-                if (ev) {
-                    u = PyObject_GetAttrString(ev, "value");
-                    Py_DECREF(ev);
-                    if (u == NULL) {
-                        if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
-                            /* some other exception */
-                            break;
-                        }
-                        PyErr_Clear();
-                    }
-                }
-                if (u == NULL) {
-                    u = Py_None;
-                    Py_INCREF(u);
-                }
-                PUSH(u);
                 continue;
             }
-            /* x is iterator, retval is value to be yielded */
+            /* x remains on stack, retval is value to be yielded */
-            f->f_yieldfrom = x;
             f->f_stacktop = stack_pointer;
             why = WHY_YIELD;
+            /* and repeat... */
+            f->f_lasti--;
             goto fast_yield;
 
         TARGET(YIELD_VALUE)

Python/compile.c

@@ -840,9 +840,9 @@ opcode_stack_effect(int opcode, int oparg)
         case IMPORT_STAR:
             return -1;
         case YIELD_VALUE:
-        case YIELD_FROM:
             return 0;
-
+        case YIELD_FROM:
+            return -1;
         case POP_BLOCK:
             return 0;
         case POP_EXCEPT:
@@ -3323,6 +3323,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
             ADDOP_O(c, LOAD_CONST, Py_None, consts);
         }
         if (e->kind == YieldFrom_kind) {
+            ADDOP(c, GET_ITER);
+            ADDOP_O(c, LOAD_CONST, Py_None, consts);
             ADDOP(c, YIELD_FROM);
         }
         else {

Python/import.c

@@ -104,7 +104,8 @@ typedef unsigned short mode_t;
        Python 3.2a2  3180 (add DELETE_DEREF)
        Python 3.3a0  3190 __class__ super closure changed
        Python 3.3a0  3200 (__qualname__ added)
-                     3210 (added size modulo 2**32 to the pyc header)
+       Python 3.3a1  3210 (added size modulo 2**32 to the pyc header)
+                     3220 (changed PEP 380 implementation)
 */
 
 /* MAGIC must change whenever the bytecode emitted by the compiler may no