traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

perform yield from delegation by repeating YIELD_FROM opcode (closes #14230) 

This allows generators that are using yield from to be seen by debuggers. It
also kills the f_yieldfrom field on frame objects.

Patch mostly from Mark Shannon with a few tweaks by me.
This commit is contained in:
Benjamin Peterson 2012-03-15 15:37:39 -05:00
parent 3270d11d8a
commit 2afe6aeae8
9 changed files with 113 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Include/frameobject.h
View File

@ -27,7 +27,6 @@ typedef struct _frame {
to the current stack top. */
PyObject **f_stacktop;
PyObject *f_trace; /* Trace function */
PyObject *f_yieldfrom; /* Iterator being delegated to by yield from */
/* In a generator, we need to be able to swap between the exception
state inside the generator and the exception state of the calling

3
Include/genobject.h
View File

@ -19,7 +19,7 @@ typedef struct {
/* True if generator is being executed. */
char gi_running;
/* The code object backing the generator */
PyObject *gi_code;
@ -35,6 +35,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
PyAPI_FUNC(int) PyGen_FetchStopIterationValue(PyObject **);
PyObject *_PyGen_Send(PyGenObject *, PyObject *);
#ifdef __cplusplus
}

23
Lib/test/test_pep380.py
View File

@ -10,7 +10,7 @@ see <http://www.cosc.canterbury.ac.nz/greg.ewing/python/yield-from/YieldFrom-Pyt
import unittest
import io
import sys
import traceback
import inspect
import parser
from test.support import captured_stderr
@ -919,6 +919,27 @@ class TestPEP380Operation(unittest.TestCase):
next(g1)
g1.close()
def test_delegator_is_visible_to_debugger(self):
def call_stack():
return [f[3] for f in inspect.stack()]
def gen():
yield call_stack()
yield call_stack()
yield call_stack()
def spam(g):
yield from g
def eggs(g):
yield from g
for stack in spam(gen()):
self.assertTrue('spam' in stack)
for stack in spam(eggs(gen())):
self.assertTrue('spam' in stack and 'eggs' in stack)
def test_main():
from test import support

2
Lib/test/test_sys.py
View File

@ -730,7 +730,7 @@ class SizeofTest(unittest.TestCase):
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
check(x, size(vh + '13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
check(x, size(vh + '12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
check(func, size(h + '12P'))

4
Objects/frameobject.c
View File

@ -444,7 +444,6 @@ frame_dealloc(PyFrameObject *f)
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
Py_CLEAR(f->f_yieldfrom);
co = f->f_code;
if (co->co_zombieframe == NULL)
@ -476,7 +475,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
Py_VISIT(f->f_exc_type);
Py_VISIT(f->f_exc_value);
Py_VISIT(f->f_exc_traceback);
Py_VISIT(f->f_yieldfrom);
/* locals */
slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@ -510,7 +508,6 @@ frame_clear(PyFrameObject *f)
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
Py_CLEAR(f->f_trace);
Py_CLEAR(f->f_yieldfrom);
/* locals */
slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@ -714,7 +711,6 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
f->f_lasti = -1;
f->f_lineno = code->co_firstlineno;
f->f_iblock = 0;
f->f_yieldfrom = NULL;
_PyObject_GC_TRACK(f);
return f;

179
Objects/genobject.c
View File

@ -6,7 +6,6 @@
#include "opcode.h"
static PyObject *gen_close(PyGenObject *gen, PyObject *args);
static void gen_undelegate(PyGenObject *gen);
static int
gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
@ -41,15 +40,6 @@ gen_dealloc(PyGenObject *gen)
PyObject_GC_Del(gen);
}
static int
gen_running(PyGenObject *gen)
{
if (gen->gi_running) {
PyErr_SetString(PyExc_ValueError, "generator already executing");
return 1;
}
return 0;
}
static PyObject *
gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
@ -58,7 +48,11 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
PyFrameObject *f = gen->gi_frame;
PyObject *result;
assert(!gen->gi_running);
if (gen->gi_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (f==NULL || f->f_stacktop == NULL) {
/* Only set exception if called from send() */
if (arg && !exc)
@ -136,45 +130,10 @@ PyDoc_STRVAR(send_doc,
"send(arg) -> send 'arg' into generator,\n\
return next yielded value or raise StopIteration.");
static PyObject *
gen_send(PyGenObject *gen, PyObject *arg)
PyObject *
_PyGen_Send(PyGenObject *gen, PyObject *arg)
{
int exc = 0;
PyObject *ret;
PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
if (gen_running(gen))
return NULL;
/* XXX (ncoghlan): Are the incref/decref on arg and yf strictly needed?
* Or would it be valid to rely on borrowed references?
*/
Py_INCREF(arg);
if (yf) {
Py_INCREF(yf);
gen->gi_running = 1;
if (PyGen_CheckExact(yf)) {
ret = gen_send((PyGenObject *)yf, arg);
} else {
if (arg == Py_None)
ret = PyIter_Next(yf);
else
ret = PyObject_CallMethod(yf, "send", "O", arg);
}
gen->gi_running = 0;
if (ret) {
Py_DECREF(yf);
goto done;
}
gen_undelegate(gen);
Py_CLEAR(arg);
if (PyGen_FetchStopIterationValue(&arg) < 0) {
exc = 1;
}
Py_DECREF(yf);
}
ret = gen_send_ex(gen, arg, exc);
done:
Py_XDECREF(arg);
return ret;
return gen_send_ex(gen, arg, 0);
}
PyDoc_STRVAR(close_doc,
@ -186,49 +145,61 @@ PyDoc_STRVAR(close_doc,
*/
static int
gen_close_iter(PyGenObject *gen, PyObject *yf)
gen_close_iter(PyObject *yf)
{
PyObject *retval = NULL;
int err = 0;
if (PyGen_CheckExact(yf)) {
retval = gen_close((PyGenObject *)yf, NULL);
if (!retval)
err = -1;
if (retval == NULL)
return -1;
} else {
PyObject *meth;
gen->gi_running = 1;
meth = PyObject_GetAttrString(yf, "close");
PyObject *meth = PyObject_GetAttrString(yf, "close");
if (meth == NULL) {
if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
if (!PyErr_ExceptionMatches(PyExc_AttributeError))
PyErr_WriteUnraisable(yf);
}
PyErr_Clear();
} else {
retval = PyObject_CallFunction(meth, "");
Py_DECREF(meth);
if (!retval)
err = -1;
if (retval == NULL)
return -1;
}
gen->gi_running = 0;
}
Py_XDECREF(retval);
return err;
}
return 0;
}
static PyObject *
gen_yf(PyGenObject *gen)
{
PyObject *yf = NULL;
PyFrameObject *f = gen->gi_frame;
if (f) {
PyObject *bytecode = f->f_code->co_code;
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
if (code[f->f_lasti + 1] != YIELD_FROM)
return NULL;
yf = f->f_stacktop[-1];
Py_INCREF(yf);
}
return yf;
}
static PyObject *
gen_close(PyGenObject *gen, PyObject *args)
{
PyObject *retval;
PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
PyObject *yf = gen_yf(gen);
int err = 0;
if (gen_running(gen))
return NULL;
if (yf) {
Py_INCREF(yf);
err = gen_close_iter(gen, yf);
gen_undelegate(gen);
gen->gi_running = 1;
err = gen_close_iter(yf);
gen->gi_running = 0;
Py_DECREF(yf);
}
if (err == 0)
@ -241,8 +212,7 @@ gen_close(PyGenObject *gen, PyObject *args)
return NULL;
}
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit))
{
|| PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
@ -323,29 +293,27 @@ gen_throw(PyGenObject *gen, PyObject *args)
PyObject *typ;
PyObject *tb = NULL;
PyObject *val = NULL;
PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
PyObject *yf = gen_yf(gen);
if (!PyArg_UnpackTuple(args, "throw", 1, 3, &typ, &val, &tb))
return NULL;
if (gen_running(gen))
return NULL;
if (yf) {
PyObject *ret;
int err;
Py_INCREF(yf);
if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
err = gen_close_iter(gen, yf);
gen->gi_running = 1;
err = gen_close_iter(yf);
gen->gi_running = 0;
Py_DECREF(yf);
gen_undelegate(gen);
if (err < 0)
return gen_send_ex(gen, Py_None, 1);
goto throw_here;
}
gen->gi_running = 1;
if (PyGen_CheckExact(yf)) {
gen->gi_running = 1;
ret = gen_throw((PyGenObject *)yf, args);
gen->gi_running = 0;
} else {
PyObject *meth = PyObject_GetAttrString(yf, "throw");
if (meth == NULL) {
@ -355,18 +323,22 @@ gen_throw(PyGenObject *gen, PyObject *args)
}
PyErr_Clear();
Py_DECREF(yf);
gen_undelegate(gen);
gen->gi_running = 0;
goto throw_here;
}
gen->gi_running = 1;
ret = PyObject_CallObject(meth, args);
gen->gi_running = 0;
Py_DECREF(meth);
}
gen->gi_running = 0;
Py_DECREF(yf);
if (!ret) {
PyObject *val;
gen_undelegate(gen);
/* Pop subiterator from stack */
ret = *(--gen->gi_frame->f_stacktop);
assert(ret == yf);
Py_DECREF(ret);
/* Termination repetition of YIELD_FROM */
gen->gi_frame->f_lasti++;
if (PyGen_FetchStopIterationValue(&val) == 0) {
ret = gen_send_ex(gen, val, 0);
Py_DECREF(val);
@ -441,44 +413,11 @@ gen_iternext(PyGenObject *gen)
{
PyObject *val = NULL;
PyObject *ret;
int exc = 0;
PyObject *yf = gen->gi_frame ? gen->gi_frame->f_yieldfrom : NULL;
if (gen_running(gen))
return NULL;
if (yf) {
Py_INCREF(yf);
/* ceval.c ensures that yf is an iterator */
gen->gi_running = 1;
ret = Py_TYPE(yf)->tp_iternext(yf);
gen->gi_running = 0;
if (ret) {
Py_DECREF(yf);
return ret;
}
gen_undelegate(gen);
if (PyGen_FetchStopIterationValue(&val) < 0)
exc = 1;
Py_DECREF(yf);
}
ret = gen_send_ex(gen, val, exc);
ret = gen_send_ex(gen, val, 0);
Py_XDECREF(val);
return ret;
}
/*
* In certain recursive situations, a generator may lose its frame
* before we get a chance to clear f_yieldfrom, so we use this
* helper function.
*/
static void
gen_undelegate(PyGenObject *gen) {
if (gen->gi_frame) {
Py_XDECREF(gen->gi_frame->f_yieldfrom);
gen->gi_frame->f_yieldfrom = NULL;
}
}
/*
* If StopIteration exception is set, fetches its 'value'
* attribute if any, otherwise sets pvalue to None.
@ -492,7 +431,7 @@ int
PyGen_FetchStopIterationValue(PyObject **pvalue) {
PyObject *et, *ev, *tb;
PyObject *value = NULL;
if (PyErr_ExceptionMatches(PyExc_StopIteration)) {
PyErr_Fetch(&et, &ev, &tb);
Py_XDECREF(et);
@ -548,7 +487,7 @@ static PyMemberDef gen_memberlist[] = {
};
static PyMethodDef gen_methods[] = {
{"send",(PyCFunction)gen_send, METH_O, send_doc},
{"send",(PyCFunction)_PyGen_Send, METH_O, send_doc},
{"throw",(PyCFunction)gen_throw, METH_VARARGS, throw_doc},
{"close",(PyCFunction)gen_close, METH_NOARGS, close_doc},
{NULL, NULL} /* Sentinel */

58
Python/ceval.c
View File

@ -1170,6 +1170,8 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
f->f_lasti to -1 (i.e. the index *before* the first instruction)
and YIELD_VALUE doesn't fiddle with f_lasti any more. So this
does work. Promise.
YIELD_FROM sets f_lasti to itself, in order to repeated yield
multiple values.
When the PREDICT() macros are enabled, some opcode pairs follow in
direct succession without updating f->f_lasti. A successful
@ -1830,49 +1832,35 @@ PyEval_EvalFrameEx(PyFrameObject *f, int throwflag)
TARGET(YIELD_FROM)
u = POP();
x = PyObject_GetIter(u);
x = TOP();
/* send u to x */
if (PyGen_CheckExact(x)) {
retval = _PyGen_Send((PyGenObject *)x, u);
} else {
if (u == Py_None)
retval = PyIter_Next(x);
else
retval = PyObject_CallMethod(x, "send", "O", u);
}
Py_DECREF(u);
if (x == NULL)
break;
/* x is now the iterator, make the first next() call */
retval = (*Py_TYPE(x)->tp_iternext)(x);
if (!retval) {
PyObject *et, *ev, *tb;
/* iter may be exhausted */
Py_CLEAR(x);
if (PyErr_Occurred() &&
!PyErr_ExceptionMatches(PyExc_StopIteration)) {
/* some other exception */
PyObject *val;
x = POP(); /* Remove iter from stack */
Py_DECREF(x);
err = PyGen_FetchStopIterationValue(&val);
if (err < 0) {
x = NULL;
break;
}
/* try to get return value from exception */
PyErr_Fetch(&et, &ev, &tb);
Py_XDECREF(et);
Py_XDECREF(tb);
/* u is return value */
u = NULL;
if (ev) {
u = PyObject_GetAttrString(ev, "value");
Py_DECREF(ev);
if (u == NULL) {
if (!PyErr_ExceptionMatches(PyExc_AttributeError)) {
/* some other exception */
break;
}
PyErr_Clear();
}
}
if (u == NULL) {
u = Py_None;
Py_INCREF(u);
}
PUSH(u);
x = val;
PUSH(x);
continue;
}
/* x is iterator, retval is value to be yielded */
f->f_yieldfrom = x;
/* x remains on stack, retval is value to be yielded */
f->f_stacktop = stack_pointer;
why = WHY_YIELD;
/* and repeat... */
f->f_lasti--;
goto fast_yield;
TARGET(YIELD_VALUE)

6
Python/compile.c
View File

@ -840,9 +840,9 @@ opcode_stack_effect(int opcode, int oparg)
case IMPORT_STAR:
return -1;
case YIELD_VALUE:
case YIELD_FROM:
return 0;
case YIELD_FROM:
return -1;
case POP_BLOCK:
return 0;
case POP_EXCEPT:
@ -3323,6 +3323,8 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
ADDOP_O(c, LOAD_CONST, Py_None, consts);
}
if (e->kind == YieldFrom_kind) {
ADDOP(c, GET_ITER);
ADDOP_O(c, LOAD_CONST, Py_None, consts);
ADDOP(c, YIELD_FROM);
}
else {

3
Python/import.c
View File

@ -104,7 +104,8 @@ typedef unsigned short mode_t;
Python 3.2a2 3180 (add DELETE_DEREF)
Python 3.3a0 3190 __class__ super closure changed
Python 3.3a0 3200 (__qualname__ added)
3210 (added size modulo 2**32 to the pyc header)
Python 3.3a1 3210 (added size modulo 2**32 to the pyc header)
3220 (changed PEP 380 implementation)
*/
/* MAGIC must change whenever the bytecode emitted by the compiler may no