traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move exc state to generator. Fixes bpo-25612 (#1773) 

Move exception state information from frame objects to coroutine (generator/thread) object where it belongs.
This commit is contained in:
Mark Shannon 2017-10-22 22:41:51 +01:00 committed by Antoine Pitrou
parent 91dc64ba3f
commit ae3087c638
13 changed files with 188 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
Include/frameobject.h
View File

@ -30,14 +30,6 @@ typedef struct _frame {
char f_trace_lines; /* Emit per-line trace events? */
char f_trace_opcodes; /* Emit per-opcode trace events? */
/* In a generator, we need to be able to swap between the exception
state inside the generator and the exception state of the calling
frame (which shouldn't be impacted when the generator "yields"
from an except handler).
These three fields exist exactly for that, and are unused for
non-generator frames. See the save_exc_state and swap_exc_state
functions in ceval.c for details of their use. */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
/* Borrowed reference to a generator, or NULL */
PyObject *f_gen;

3
Include/genobject.h
View File

@ -25,7 +25,8 @@ struct _frame; /* Avoid including frameobject.h */
/* Name of the generator. */ \
PyObject *prefix##_name; \
/* Qualified name of the generator. */ \
PyObject *prefix##_qualname;
PyObject *prefix##_qualname; \
_PyErr_StackItem prefix##_exc_state;
typedef struct {
/* The gi_ prefix is intended to remind of generator-iterator. */

1
Include/pyerrors.h
View File

@ -78,6 +78,7 @@ PyAPI_FUNC(void) PyErr_SetNone(PyObject *);
PyAPI_FUNC(void) PyErr_SetObject(PyObject *, PyObject *);
#ifndef Py_LIMITED_API
PyAPI_FUNC(void) _PyErr_SetKeyError(PyObject *);
_PyErr_StackItem *_PyErr_GetTopmostException(PyThreadState *tstate);
#endif
PyAPI_FUNC(void) PyErr_SetString(
PyObject *exception,

27
Include/pystate.h
View File

@ -123,6 +123,21 @@ typedef int (*Py_tracefunc)(PyObject *, struct _frame *, int, PyObject *);
#ifdef Py_LIMITED_API
typedef struct _ts PyThreadState;
#else
typedef struct _err_stackitem {
/* This struct represents an entry on the exception stack, which is a
* per-coroutine state. (Coroutine in the computer science sense,
* including the thread and generators).
* This ensures that the exception state is not impacted by "yields"
* from an except handler.
*/
PyObject *exc_type, *exc_value, *exc_traceback;
struct _err_stackitem *previous_item;
} _PyErr_StackItem;
typedef struct _ts {
/* See Python/ceval.c for comments explaining most fields */
@ -147,13 +162,19 @@ typedef struct _ts {
PyObject *c_profileobj;
PyObject *c_traceobj;
/* The exception currently being raised */
PyObject *curexc_type;
PyObject *curexc_value;
PyObject *curexc_traceback;
PyObject *exc_type;
PyObject *exc_value;
PyObject *exc_traceback;
/* The exception currently being handled, if no coroutines/generators
* are present. Always last element on the stack referred to be exc_info.
*/
_PyErr_StackItem exc_state;
/* Pointer to the top of the stack of the exceptions currently
* being handled */
_PyErr_StackItem *exc_info;
PyObject *dict; /* Stores per-thread state */

56
Lib/test/test_exceptions.py
View File

@ -1097,6 +1097,62 @@ class ExceptionTests(unittest.TestCase):
self.assertIn("test message", report)
self.assertTrue(report.endswith("\n"))
def test_yield_in_nested_try_excepts(self):
#Issue #25612
class MainError(Exception):
pass
class SubError(Exception):
pass
def main():
try:
raise MainError()
except MainError:
try:
yield
except SubError:
pass
raise
coro = main()
coro.send(None)
with self.assertRaises(MainError):
coro.throw(SubError())
def test_generator_doesnt_retain_old_exc2(self):
#Issue 28884#msg282532
def g():
try:
raise ValueError
except ValueError:
yield 1
self.assertEqual(sys.exc_info(), (None, None, None))
yield 2
gen = g()
try:
raise IndexError
except IndexError:
self.assertEqual(next(gen), 1)
self.assertEqual(next(gen), 2)
def test_raise_in_generator(self):
#Issue 25612#msg304117
def g():
yield 1
raise
yield 2
with self.assertRaises(ZeroDivisionError):
i = g()
try:
1/0
except:
next(i)
next(i)
class ImportErrorTests(unittest.TestCase):

4
Lib/test/test_sys.py
View File

@ -971,7 +971,7 @@ class SizeofTest(unittest.TestCase):
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
check(x, vsize('8P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
check(x, vsize('5P2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
check(func, size('12P'))
@ -988,7 +988,7 @@ class SizeofTest(unittest.TestCase):
check(bar, size('PP'))
# generator
def get_gen(): yield 1
check(get_gen(), size('Pb2PPP'))
check(get_gen(), size('Pb2PPP4P'))
# iterator
check(iter('abc'), size('lP'))
# callable-iterator

3
Misc/NEWS.d/next/C API/2017-10-22-13-12-28.bpo-25612.1jnWKT.rst Normal file
View File

@ -0,0 +1,3 @@
Move the current exception state from the frame object to the co-routine.
This simplifies the interpreter and fixes a couple of obscure bugs caused by
having swap exception state when entering or exiting a generator.

13
Objects/frameobject.c
View File

@ -379,8 +379,7 @@ static PyGetSetDef frame_getsetlist[] = {
* ob_type, ob_size, f_code, f_valuestack;
* f_locals, f_trace,
f_exc_type, f_exc_value, f_exc_traceback are NULL;
* f_locals, f_trace are NULL;
* f_localsplus does not require re-allocation and
the local variables in f_localsplus are NULL.
@ -438,9 +437,6 @@ frame_dealloc(PyFrameObject *f)
Py_DECREF(f->f_globals);
Py_CLEAR(f->f_locals);
Py_CLEAR(f->f_trace);
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
co = f->f_code;
if (co->co_zombieframe == NULL)
@ -469,9 +465,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
Py_VISIT(f->f_globals);
Py_VISIT(f->f_locals);
Py_VISIT(f->f_trace);
Py_VISIT(f->f_exc_type);
Py_VISIT(f->f_exc_value);
Py_VISIT(f->f_exc_traceback);
/* locals */
slots = f->f_code->co_nlocals + PyTuple_GET_SIZE(f->f_code->co_cellvars) + PyTuple_GET_SIZE(f->f_code->co_freevars);
@ -502,9 +495,6 @@ frame_tp_clear(PyFrameObject *f)
f->f_stacktop = NULL;
f->f_executing = 0;
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
Py_CLEAR(f->f_trace);
/* locals */
@ -698,7 +688,6 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code,
f->f_localsplus[i] = NULL;
f->f_locals = NULL;
f->f_trace = NULL;
f->f_exc_type = f->f_exc_value = f->f_exc_traceback = NULL;
}
f->f_stacktop = f->f_valuestack;
f->f_builtins = builtins;

46
Objects/genobject.c
View File

@ -16,6 +16,15 @@ static char *NON_INIT_CORO_MSG = "can't send non-None value to a "
static char *ASYNC_GEN_IGNORED_EXIT_MSG =
"async generator ignored GeneratorExit";
static inline int
exc_state_traverse(_PyErr_StackItem *exc_state, visitproc visit, void *arg)
{
Py_VISIT(exc_state->exc_type);
Py_VISIT(exc_state->exc_value);
Py_VISIT(exc_state->exc_traceback);
return 0;
}
static int
gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
{
@ -23,7 +32,7 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg)
Py_VISIT(gen->gi_code);
Py_VISIT(gen->gi_name);
Py_VISIT(gen->gi_qualname);
return 0;
return exc_state_traverse(&gen->gi_exc_state, visit, arg);
}
void
@ -87,6 +96,21 @@ _PyGen_Finalize(PyObject *self)
PyErr_Restore(error_type, error_value, error_traceback);
}
static inline void
exc_state_clear(_PyErr_StackItem *exc_state)
{
PyObject *t, *v, *tb;
t = exc_state->exc_type;
v = exc_state->exc_value;
tb = exc_state->exc_traceback;
exc_state->exc_type = NULL;
exc_state->exc_value = NULL;
exc_state->exc_traceback = NULL;
Py_XDECREF(t);
Py_XDECREF(v);
Py_XDECREF(tb);
}
static void
gen_dealloc(PyGenObject *gen)
{
@ -116,6 +140,7 @@ gen_dealloc(PyGenObject *gen)
Py_CLEAR(gen->gi_code);
Py_CLEAR(gen->gi_name);
Py_CLEAR(gen->gi_qualname);
exc_state_clear(&gen->gi_exc_state);
PyObject_GC_Del(gen);
}
@ -187,7 +212,11 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing)
f->f_back = tstate->frame;
gen->gi_running = 1;
gen->gi_exc_state.previous_item = tstate->exc_info;
tstate->exc_info = &gen->gi_exc_state;
result = PyEval_EvalFrameEx(f, exc);
tstate->exc_info = gen->gi_exc_state.previous_item;
gen->gi_exc_state.previous_item = NULL;
gen->gi_running = 0;
/* Don't keep the reference to f_back any longer than necessary. It
@ -281,16 +310,7 @@ gen_send_ex(PyGenObject *gen, PyObject *arg, int exc, int closing)
if (!result || f->f_stacktop == NULL) {
/* generator can't be rerun, so release the frame */
/* first clean reference cycle through stored exception traceback */
PyObject *t, *v, *tb;
t = f->f_exc_type;
v = f->f_exc_value;
tb = f->f_exc_traceback;
f->f_exc_type = NULL;
f->f_exc_value = NULL;
f->f_exc_traceback = NULL;
Py_XDECREF(t);
Py_XDECREF(v);
Py_XDECREF(tb);
exc_state_clear(&gen->gi_exc_state);
gen->gi_frame->f_gen = NULL;
gen->gi_frame = NULL;
Py_DECREF(f);
@ -810,6 +830,10 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f,
gen->gi_code = (PyObject *)(f->f_code);
gen->gi_running = 0;
gen->gi_weakreflist = NULL;
gen->gi_exc_state.exc_type = NULL;
gen->gi_exc_state.exc_value = NULL;
gen->gi_exc_state.exc_traceback = NULL;
gen->gi_exc_state.previous_item = NULL;
if (name != NULL)
gen->gi_name = name;
else

125
Python/ceval.c
View File

@ -511,9 +511,6 @@ enum why_code {
WHY_SILENCED = 0x0080 /* Exception silenced by 'with' */
};
static void save_exc_state(PyThreadState *, PyFrameObject *);
static void swap_exc_state(PyThreadState *, PyFrameObject *);
static void restore_and_clear_exc_state(PyThreadState *, PyFrameObject *);
static int do_raise(PyObject *, PyObject *);
static int unpack_iterable(PyObject *, int, int, PyObject **);
@ -813,17 +810,19 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
#define UNWIND_EXCEPT_HANDLER(b) \
do { \
PyObject *type, *value, *traceback; \
_PyErr_StackItem *exc_info; \
assert(STACK_LEVEL() >= (b)->b_level + 3); \
while (STACK_LEVEL() > (b)->b_level + 3) { \
value = POP(); \
Py_XDECREF(value); \
} \
type = tstate->exc_type; \
value = tstate->exc_value; \
traceback = tstate->exc_traceback; \
tstate->exc_type = POP(); \
tstate->exc_value = POP(); \
tstate->exc_traceback = POP(); \
exc_info = tstate->exc_info; \
type = exc_info->exc_type; \
value = exc_info->exc_value; \
traceback = exc_info->exc_traceback; \
exc_info->exc_type = POP(); \
exc_info->exc_value = POP(); \
exc_info->exc_traceback = POP(); \
Py_XDECREF(type); \
Py_XDECREF(value); \
Py_XDECREF(traceback); \
@ -910,16 +909,6 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag)
f->f_stacktop = NULL; /* remains NULL unless yield suspends frame */
f->f_executing = 1;
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
if (!throwflag && f->f_exc_type != NULL && f->f_exc_type != Py_None) {
/* We were in an except handler when we left,
restore the exception state which was put aside
(see YIELD_VALUE). */
swap_exc_state(tstate, f);
}
else
save_exc_state(tstate, f);
}
#ifdef LLTRACE
lltrace = _PyDict_GetItemId(f->f_globals, &PyId___ltrace__) != NULL;
@ -3447,12 +3436,13 @@ fast_block_end:
|| b->b_type == SETUP_FINALLY)) {
PyObject *exc, *val, *tb;
int handler = b->b_handler;
_PyErr_StackItem *exc_info = tstate->exc_info;
/* Beware, this invalidates all b->b_* fields */
PyFrame_BlockSetup(f, EXCEPT_HANDLER, -1, STACK_LEVEL());
PUSH(tstate->exc_traceback);
PUSH(tstate->exc_value);
if (tstate->exc_type != NULL) {
PUSH(tstate->exc_type);
PUSH(exc_info->exc_traceback);
PUSH(exc_info->exc_value);
if (exc_info->exc_type != NULL) {
PUSH(exc_info->exc_type);
}
else {
Py_INCREF(Py_None);
@ -3470,10 +3460,10 @@ fast_block_end:
else
PyException_SetTraceback(val, Py_None);
Py_INCREF(exc);
tstate->exc_type = exc;
exc_info->exc_type = exc;
Py_INCREF(val);
tstate->exc_value = val;
tstate->exc_traceback = tb;
exc_info->exc_value = val;
exc_info->exc_traceback = tb;
if (tb == NULL)
tb = Py_None;
Py_INCREF(tb);
@ -3516,28 +3506,6 @@ fast_block_end:
assert((retval != NULL) ^ (PyErr_Occurred() != NULL));
fast_yield:
if (co->co_flags & (CO_GENERATOR | CO_COROUTINE | CO_ASYNC_GENERATOR)) {
/* The purpose of this block is to put aside the generator's exception
state and restore that of the calling frame. If the current
exception state is from the caller, we clear the exception values
on the generator frame, so they are not swapped back in latter. The
origin of the current exception state is determined by checking for
except handler blocks, which we must be in iff a new exception
state came into existence in this frame. (An uncaught exception
would have why == WHY_EXCEPTION, and we wouldn't be here). */
int i;
for (i = 0; i < f->f_iblock; i++) {
if (f->f_blockstack[i].b_type == EXCEPT_HANDLER) {
break;
}
}
if (i == f->f_iblock)
/* We did not create this exception. */
restore_and_clear_exc_state(tstate, f);
else
swap_exc_state(tstate, f);
}
if (tstate->use_tracing) {
if (tstate->c_tracefunc) {
@ -4057,60 +4025,6 @@ special_lookup(PyObject *o, _Py_Identifier *id)
}
/* These 3 functions deal with the exception state of generators. */
static void
save_exc_state(PyThreadState *tstate, PyFrameObject *f)
{
PyObject *type, *value, *traceback;
Py_XINCREF(tstate->exc_type);
Py_XINCREF(tstate->exc_value);
Py_XINCREF(tstate->exc_traceback);
type = f->f_exc_type;
value = f->f_exc_value;
traceback = f->f_exc_traceback;
f->f_exc_type = tstate->exc_type;
f->f_exc_value = tstate->exc_value;
f->f_exc_traceback = tstate->exc_traceback;
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(traceback);
}
static void
swap_exc_state(PyThreadState *tstate, PyFrameObject *f)
{
PyObject *tmp;
tmp = tstate->exc_type;
tstate->exc_type = f->f_exc_type;
f->f_exc_type = tmp;
tmp = tstate->exc_value;
tstate->exc_value = f->f_exc_value;
f->f_exc_value = tmp;
tmp = tstate->exc_traceback;
tstate->exc_traceback = f->f_exc_traceback;
f->f_exc_traceback = tmp;
}
static void
restore_and_clear_exc_state(PyThreadState *tstate, PyFrameObject *f)
{
PyObject *type, *value, *tb;
type = tstate->exc_type;
value = tstate->exc_value;
tb = tstate->exc_traceback;
tstate->exc_type = f->f_exc_type;
tstate->exc_value = f->f_exc_value;
tstate->exc_traceback = f->f_exc_traceback;
f->f_exc_type = NULL;
f->f_exc_value = NULL;
f->f_exc_traceback = NULL;
Py_XDECREF(type);
Py_XDECREF(value);
Py_XDECREF(tb);
}
/* Logic for the raise statement (too complicated for inlining).
This *consumes* a reference count to each of its arguments. */
static int
@ -4121,10 +4035,11 @@ do_raise(PyObject *exc, PyObject *cause)
if (exc == NULL) {
/* Reraise */
PyThreadState *tstate = PyThreadState_GET();
_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
PyObject *tb;
type = tstate->exc_type;
value = tstate->exc_value;
tb = tstate->exc_traceback;
type = exc_info->exc_type;
value = exc_info->exc_value;
tb = exc_info->exc_traceback;
if (type == Py_None || type == NULL) {
PyErr_SetString(PyExc_RuntimeError,
"No active exception to reraise");

34
Python/errors.c
View File

@ -53,6 +53,18 @@ PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback)
Py_XDECREF(oldtraceback);
}
_PyErr_StackItem *
_PyErr_GetTopmostException(PyThreadState *tstate)
{
_PyErr_StackItem *exc_info = tstate->exc_info;
while ((exc_info->exc_type == NULL || exc_info->exc_type == Py_None) &&
exc_info->previous_item != NULL)
{
exc_info = exc_info->previous_item;
}
return exc_info;
}
static PyObject*
_PyErr_CreateException(PyObject *exception, PyObject *value)
{
@ -83,7 +95,7 @@ PyErr_SetObject(PyObject *exception, PyObject *value)
}
Py_XINCREF(value);
exc_value = tstate->exc_value;
exc_value = _PyErr_GetTopmostException(tstate)->exc_value;
if (exc_value != NULL && exc_value != Py_None) {
/* Implicit exception chaining */
Py_INCREF(exc_value);
@ -335,9 +347,11 @@ PyErr_GetExcInfo(PyObject **p_type, PyObject **p_value, PyObject **p_traceback)
{
PyThreadState *tstate = PyThreadState_GET();
*p_type = tstate->exc_type;
*p_value = tstate->exc_value;
*p_traceback = tstate->exc_traceback;
_PyErr_StackItem *exc_info = _PyErr_GetTopmostException(tstate);
*p_type = exc_info->exc_type;
*p_value = exc_info->exc_value;
*p_traceback = exc_info->exc_traceback;
Py_XINCREF(*p_type);
Py_XINCREF(*p_value);
@ -350,13 +364,13 @@ PyErr_SetExcInfo(PyObject *p_type, PyObject *p_value, PyObject *p_traceback)
PyObject *oldtype, *oldvalue, *oldtraceback;
PyThreadState *tstate = PyThreadState_GET();
oldtype = tstate->exc_type;
oldvalue = tstate->exc_value;
oldtraceback = tstate->exc_traceback;
oldtype = tstate->exc_info->exc_type;
oldvalue = tstate->exc_info->exc_value;
oldtraceback = tstate->exc_info->exc_traceback;
tstate->exc_type = p_type;
tstate->exc_value = p_value;
tstate->exc_traceback = p_traceback;
tstate->exc_info->exc_type = p_type;
tstate->exc_info->exc_value = p_value;
tstate->exc_info->exc_traceback = p_traceback;
Py_XDECREF(oldtype);
Py_XDECREF(oldvalue);

21
Python/pystate.c
View File

@ -257,9 +257,11 @@ new_threadstate(PyInterpreterState *interp, int init)
tstate->curexc_value = NULL;
tstate->curexc_traceback = NULL;
tstate->exc_type = NULL;
tstate->exc_value = NULL;
tstate->exc_traceback = NULL;
tstate->exc_state.exc_type = NULL;
tstate->exc_state.exc_value = NULL;
tstate->exc_state.exc_traceback = NULL;
tstate->exc_state.previous_item = NULL;
tstate->exc_info = &tstate->exc_state;
tstate->c_profilefunc = NULL;
tstate->c_tracefunc = NULL;
@ -444,9 +446,16 @@ PyThreadState_Clear(PyThreadState *tstate)
Py_CLEAR(tstate->curexc_value);
Py_CLEAR(tstate->curexc_traceback);
Py_CLEAR(tstate->exc_type);
Py_CLEAR(tstate->exc_value);
Py_CLEAR(tstate->exc_traceback);
Py_CLEAR(tstate->exc_state.exc_type);
Py_CLEAR(tstate->exc_state.exc_value);
Py_CLEAR(tstate->exc_state.exc_traceback);
/* The stack of exception states should contain just this thread. */
assert(tstate->exc_info->previous_item == NULL);
if (Py_VerboseFlag && tstate->exc_info != &tstate->exc_state) {
fprintf(stderr,
"PyThreadState_Clear: warning: thread still has a generator\n");
}
tstate->c_profilefunc = NULL;
tstate->c_tracefunc = NULL;

11
Python/sysmodule.c
View File

@ -311,14 +311,13 @@ PyDoc_STRVAR(excepthook_doc,
static PyObject *
sys_exc_info(PyObject *self, PyObject *noargs)
{
PyThreadState *tstate;
tstate = PyThreadState_GET();
_PyErr_StackItem *err_info = _PyErr_GetTopmostException(PyThreadState_GET());
return Py_BuildValue(
"(OOO)",
tstate->exc_type != NULL ? tstate->exc_type : Py_None,
tstate->exc_value != NULL ? tstate->exc_value : Py_None,
tstate->exc_traceback != NULL ?
tstate->exc_traceback : Py_None);
err_info->exc_type != NULL ? err_info->exc_type : Py_None,
err_info->exc_value != NULL ? err_info->exc_value : Py_None,
err_info->exc_traceback != NULL ?
err_info->exc_traceback : Py_None);
}
PyDoc_STRVAR(exc_info_doc,