Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Issue #17807: Generators can now be finalized even when they are part of a reference cycle.

This commit is contained in:
Antoine Pitrou 2013-05-08 18:12:35 +02:00
parent 070cb3c9be
commit 04e70d19e7
8 changed files with 335 additions and 245 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
Include/frameobject.h
View File

@ -36,6 +36,8 @@ typedef struct _frame {
non-generator frames. See the save_exc_state and swap_exc_state
functions in ceval.c for details of their use. */
PyObject *f_exc_type, *f_exc_value, *f_exc_traceback;
/* Borrowed referenced to a generator, or NULL */
PyObject *f_gen;
PyThreadState *f_tstate;
int f_lasti; /* Last instruction if called */
@ -84,6 +86,13 @@ PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out);
/* Return the line of code the frame is currently executing. */
PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *);
/* Generator support */
PyAPI_FUNC(PyObject *) _PyFrame_YieldingFrom(PyFrameObject *);
PyAPI_FUNC(PyObject *) _PyFrame_GeneratorSend(PyFrameObject *, PyObject *, int exc);
PyAPI_FUNC(PyObject *) _PyFrame_Finalize(PyFrameObject *);
PyAPI_FUNC(int) _PyFrame_CloseIterator(PyObject *);
#ifdef __cplusplus
}
#endif

1
Include/genobject.h
View File

@ -33,6 +33,7 @@ PyAPI_DATA(PyTypeObject) PyGen_Type;
#define PyGen_CheckExact(op) (Py_TYPE(op) == &PyGen_Type)
PyAPI_FUNC(PyObject *) PyGen_New(struct _frame *);
/* Deprecated, kept for backwards compatibility. */
PyAPI_FUNC(int) PyGen_NeedsFinalizing(PyGenObject *);
PyAPI_FUNC(int) _PyGen_FetchStopIterationValue(PyObject **);
PyObject *_PyGen_Send(PyGenObject *, PyObject *);

53
Lib/test/test_generators.py
View File

@ -1,3 +1,55 @@
import gc
import sys
import unittest
import weakref
from test import support
class FinalizationTest(unittest.TestCase):
def test_frame_resurrect(self):
# A generator frame can be resurrected by a generator's finalization.
def gen():
nonlocal frame
try:
yield
finally:
frame = sys._getframe()
g = gen()
wr = weakref.ref(g)
next(g)
del g
support.gc_collect()
self.assertIs(wr(), None)
self.assertTrue(frame)
del frame
support.gc_collect()
def test_refcycle(self):
# A generator caught in a refcycle gets finalized anyway.
old_garbage = gc.garbage[:]
finalized = False
def gen():
nonlocal finalized
try:
g = yield
yield 1
finally:
finalized = True
g = gen()
next(g)
g.send(g)
self.assertGreater(sys.getrefcount(g), 2)
self.assertFalse(finalized)
del g
support.gc_collect()
self.assertTrue(finalized)
self.assertEqual(gc.garbage, old_garbage)
tutorial_tests = """
Let's try a simple generator:
@ -1880,6 +1932,7 @@ __test__ = {"tut": tutorial_tests,
# so this works as expected in both ways of running regrtest.
def test_main(verbose=None):
from test import support, test_generators
support.run_unittest(__name__)
support.run_doctest(test_generators, verbose)
# This part isn't needed for regrtest, but for running the test directly.

2
Lib/test/test_sys.py
View File

@ -764,7 +764,7 @@ class SizeofTest(unittest.TestCase):
nfrees = len(x.f_code.co_freevars)
extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
ncells + nfrees - 1
check(x, vsize('12P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
check(x, vsize('13P3i' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
# function
def func(): pass
check(func, size('12P'))

3
Misc/NEWS
View File

@ -10,6 +10,9 @@ What's New in Python 3.4.0 Alpha 1?
Core and Builtins
-----------------
- Issue #17807: Generators can now be finalized even when they are part of
a reference cycle.
- Issue #1545463: At shutdown, defer finalization of codec modules so
that stderr remains usable.

5
Modules/gcmodule.c
View File

@ -524,10 +524,7 @@ untrack_dicts(PyGC_Head *head)
static int
has_finalizer(PyObject *op)
{
if (PyGen_CheckExact(op))
return PyGen_NeedsFinalizing((PyGenObject *)op);
else
return op->ob_type->tp_del != NULL;
return op->ob_type->tp_del != NULL;
}
/* Move the objects in unreachable with __del__ methods into `finalizers`.

254
Objects/frameobject.c
View File

@ -31,6 +31,195 @@ frame_getlocals(PyFrameObject *f, void *closure)
return f->f_locals;
}
/*
* Generator support.
*/
PyObject *
_PyFrame_YieldingFrom(PyFrameObject *f)
{
PyObject *yf = NULL;
if (f && f->f_stacktop) {
PyObject *bytecode = f->f_code->co_code;
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
if (code[f->f_lasti + 1] != YIELD_FROM)
return NULL;
yf = f->f_stacktop[-1];
Py_INCREF(yf);
}
return yf;
}
PyObject *
_PyFrame_GeneratorSend(PyFrameObject *f, PyObject *arg, int exc)
{
PyThreadState *tstate = PyThreadState_GET();
PyObject *result;
PyGenObject *gen = (PyGenObject *) f->f_gen;
assert(gen == NULL || PyGen_CheckExact(gen));
if (gen && gen->gi_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (f->f_stacktop == NULL) {
/* Only set exception if send() called, not throw() or next() */
if (arg && !exc)
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
if (f->f_lasti == -1) {
if (arg && arg != Py_None) {
PyErr_SetString(PyExc_TypeError,
"can't send non-None value to a "
"just-started generator");
return NULL;
}
} else {
/* Push arg onto the frame's value stack */
result = arg ? arg : Py_None;
Py_INCREF(result);
*(f->f_stacktop++) = result;
}
/* Generators always return to their most recent caller, not
* necessarily their creator. */
Py_XINCREF(tstate->frame);
assert(f->f_back == NULL);
f->f_back = tstate->frame;
if (gen) {
Py_INCREF(gen);
gen->gi_running = 1;
}
result = PyEval_EvalFrameEx(f, exc);
if (gen) {
gen->gi_running = 0;
/* In case running the frame has lost all external references
* to gen, we must be careful not to hold on an invalid object. */
if (Py_REFCNT(gen) == 1)
Py_CLEAR(gen);
else
Py_DECREF(gen);
}
/* Don't keep the reference to f_back any longer than necessary. It
* may keep a chain of frames alive or it could create a reference
* cycle. */
assert(f->f_back == tstate->frame);
Py_CLEAR(f->f_back);
/* If the generator just returned (as opposed to yielding), signal
* that the generator is exhausted. */
if (result && f->f_stacktop == NULL) {
if (result == Py_None) {
/* Delay exception instantiation if we can */
PyErr_SetNone(PyExc_StopIteration);
} else {
PyObject *e = PyObject_CallFunctionObjArgs(
PyExc_StopIteration, result, NULL);
if (e != NULL) {
PyErr_SetObject(PyExc_StopIteration, e);
Py_DECREF(e);
}
}
Py_CLEAR(result);
}
if (f->f_stacktop == NULL) {
/* generator can't be rerun, so release the frame */
/* first clean reference cycle through stored exception traceback */
PyObject *t, *v, *tb;
t = f->f_exc_type;
v = f->f_exc_value;
tb = f->f_exc_traceback;
f->f_exc_type = NULL;
f->f_exc_value = NULL;
f->f_exc_traceback = NULL;
Py_XDECREF(t);
Py_XDECREF(v);
Py_XDECREF(tb);
if (gen) {
f->f_gen = NULL;
Py_CLEAR(gen->gi_frame);
}
}
return result;
}
int
_PyFrame_CloseIterator(PyObject *yf)
{
PyObject *retval = NULL;
_Py_IDENTIFIER(close);
if (PyGen_CheckExact(yf)) {
PyFrameObject *f = ((PyGenObject *) yf)->gi_frame;
assert(f != NULL);
retval = _PyFrame_Finalize(f);
if (retval == NULL)
return -1;
} else {
PyObject *meth = _PyObject_GetAttrId(yf, &PyId_close);
if (meth == NULL) {
if (!PyErr_ExceptionMatches(PyExc_AttributeError))
PyErr_WriteUnraisable(yf);
PyErr_Clear();
} else {
retval = PyObject_CallFunction(meth, "");
Py_DECREF(meth);
if (retval == NULL)
return -1;
}
}
Py_XDECREF(retval);
return 0;
}
PyObject *
_PyFrame_Finalize(PyFrameObject *f)
{
int err = 0;
PyObject *retval;
PyGenObject *gen = (PyGenObject *) f->f_gen;
PyObject *yf = _PyFrame_YieldingFrom(f);
assert(gen == NULL || PyGen_CheckExact(gen));
if (yf) {
if (gen)
gen->gi_running = 1;
err = _PyFrame_CloseIterator(yf);
if (gen)
gen->gi_running = 0;
Py_DECREF(yf);
}
if (err == 0)
PyErr_SetNone(PyExc_GeneratorExit);
retval = _PyFrame_GeneratorSend(f, Py_None, 1);
if (retval) {
Py_DECREF(retval);
PyErr_SetString(PyExc_RuntimeError,
"generator ignored GeneratorExit");
return NULL;
}
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
}
return NULL;
}
/*
* Line number support.
*/
int
PyFrame_GetLineNumber(PyFrameObject *f)
{
@ -419,33 +608,44 @@ static int numfree = 0; /* number of frames currently in free_list */
/* max value for numfree */
#define PyFrame_MAXFREELIST 200
static void
frame_clear(PyFrameObject *f);
static void
frame_dealloc(PyFrameObject *f)
{
PyObject **p, **valuestack;
PyCodeObject *co;
Py_REFCNT(f)++;
frame_clear(f);
Py_REFCNT(f)--;
if (Py_REFCNT(f) > 0) {
/* Frame resurrected! */
Py_ssize_t refcnt = Py_REFCNT(f);
_Py_NewReference((PyObject *) f);
Py_REFCNT(f) = refcnt;
/* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
* we need to undo that. */
_Py_DEC_REFTOTAL;
/* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
* chain, so no more to do there.
* If COUNT_ALLOCS, the original decref bumped tp_frees, and
* _Py_NewReference bumped tp_allocs: both of those need to be
* undone.
*/
#ifdef COUNT_ALLOCS
--(Py_TYPE(self)->tp_frees);
--(Py_TYPE(self)->tp_allocs);
#endif
}
PyObject_GC_UnTrack(f);
Py_TRASHCAN_SAFE_BEGIN(f)
/* Kill all local variables */
valuestack = f->f_valuestack;
for (p = f->f_localsplus; p < valuestack; p++)
Py_CLEAR(*p);
/* Free stack */
if (f->f_stacktop != NULL) {
for (p = valuestack; p < f->f_stacktop; p++)
Py_XDECREF(*p);
}
Py_XDECREF(f->f_back);
Py_DECREF(f->f_builtins);
Py_DECREF(f->f_globals);
Py_CLEAR(f->f_locals);
Py_CLEAR(f->f_trace);
Py_CLEAR(f->f_exc_type);
Py_CLEAR(f->f_exc_value);
Py_CLEAR(f->f_exc_traceback);
co = f->f_code;
if (co->co_zombieframe == NULL)
@ -497,12 +697,25 @@ frame_clear(PyFrameObject *f)
{
PyObject **fastlocals, **p, **oldtop;
Py_ssize_t i, slots;
PyObject *retval;
/* Before anything else, make sure that this frame is clearly marked
* as being defunct! Else, e.g., a generator reachable from this
* frame may also point to this frame, believe itself to still be
* active, and try cleaning up this frame again.
*/
if (f->f_back == NULL) {
PyObject *t, *v, *tb;
PyErr_Fetch(&t, &v, &tb);
/* Note that this can finalize a suspended generator frame even
* if the generator object was disposed of (i.e. if f_gen is NULL).
*/
retval = _PyFrame_Finalize(f);
if (retval == NULL) {
if (PyErr_Occurred())
PyErr_WriteUnraisable((PyObject *) f);
}
else
Py_DECREF(retval);
PyErr_Restore(t, v, tb);
}
/* Make sure the frame is now clearly marked as being defunct */
oldtop = f->f_stacktop;
f->f_stacktop = NULL;
@ -713,6 +926,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, PyObject *globals,
f->f_lasti = -1;
f->f_lineno = code->co_firstlineno;
f->f_iblock = 0;
f->f_gen = NULL;
_PyObject_GC_TRACK(f);
return f;

253
Objects/genobject.c
View File

@ -19,112 +19,50 @@ static void
gen_dealloc(PyGenObject *gen)
{
PyObject *self = (PyObject *) gen;
PyFrameObject *f = gen->gi_frame;
_PyObject_GC_UNTRACK(gen);
if (gen->gi_weakreflist != NULL)
PyObject_ClearWeakRefs(self);
_PyObject_GC_TRACK(self);
if (gen->gi_frame != NULL && gen->gi_frame->f_stacktop != NULL) {
/* Generator is paused, so we need to close */
Py_TYPE(gen)->tp_del(self);
if (self->ob_refcnt > 0)
return; /* resurrected. :( */
gen->gi_frame = NULL;
if (f) {
/* Close the generator by finalizing the frame */
PyObject *retval, *t, *v, *tb;
PyErr_Fetch(&t, &v, &tb);
f->f_gen = NULL;
retval = _PyFrame_Finalize(f);
if (retval)
Py_DECREF(retval);
else if (PyErr_Occurred())
PyErr_WriteUnraisable((PyObject *) gen);
Py_DECREF(f);
PyErr_Restore(t, v, tb);
}
_PyObject_GC_UNTRACK(self);
Py_CLEAR(gen->gi_frame);
Py_CLEAR(gen->gi_code);
PyObject_GC_Del(gen);
}
static PyObject *
gen_send_ex(PyGenObject *gen, PyObject *arg, int exc)
{
PyThreadState *tstate = PyThreadState_GET();
PyFrameObject *f = gen->gi_frame;
PyObject *result;
/* For compatibility, we check gi_running before f == NULL */
if (gen->gi_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (f == NULL || f->f_stacktop == NULL) {
/* Only set exception if called from send() */
if (f == NULL) {
/* Only set exception if send() called, not throw() or next() */
if (arg && !exc)
PyErr_SetNone(PyExc_StopIteration);
return NULL;
}
if (f->f_lasti == -1) {
if (arg && arg != Py_None) {
PyErr_SetString(PyExc_TypeError,
"can't send non-None value to a "
"just-started generator");
return NULL;
}
} else {
/* Push arg onto the frame's value stack */
result = arg ? arg : Py_None;
Py_INCREF(result);
*(f->f_stacktop++) = result;
}
/* Generators always return to their most recent caller, not
* necessarily their creator. */
Py_XINCREF(tstate->frame);
assert(f->f_back == NULL);
f->f_back = tstate->frame;
gen->gi_running = 1;
result = PyEval_EvalFrameEx(f, exc);
gen->gi_running = 0;
/* Don't keep the reference to f_back any longer than necessary. It
* may keep a chain of frames alive or it could create a reference
* cycle. */
assert(f->f_back == tstate->frame);
Py_CLEAR(f->f_back);
/* If the generator just returned (as opposed to yielding), signal
* that the generator is exhausted. */
if (result && f->f_stacktop == NULL) {
if (result == Py_None) {
/* Delay exception instantiation if we can */
PyErr_SetNone(PyExc_StopIteration);
} else {
PyObject *e = PyObject_CallFunctionObjArgs(
PyExc_StopIteration, result, NULL);
if (e != NULL) {
PyErr_SetObject(PyExc_StopIteration, e);
Py_DECREF(e);
}
}
Py_CLEAR(result);
}
if (!result || f->f_stacktop == NULL) {
/* generator can't be rerun, so release the frame */
/* first clean reference cycle through stored exception traceback */
PyObject *t, *v, *tb;
t = f->f_exc_type;
v = f->f_exc_value;
tb = f->f_exc_traceback;
f->f_exc_type = NULL;
f->f_exc_value = NULL;
f->f_exc_traceback = NULL;
Py_XDECREF(t);
Py_XDECREF(v);
Py_XDECREF(tb);
gen->gi_frame = NULL;
Py_DECREF(f);
}
return result;
return _PyFrame_GeneratorSend(f, arg, exc);
}
PyDoc_STRVAR(send_doc,
@ -145,146 +83,33 @@ PyDoc_STRVAR(close_doc,
* close a subiterator being delegated to by yield-from.
*/
static int
gen_close_iter(PyObject *yf)
{
PyObject *retval = NULL;
_Py_IDENTIFIER(close);
if (PyGen_CheckExact(yf)) {
retval = gen_close((PyGenObject *)yf, NULL);
if (retval == NULL)
return -1;
} else {
PyObject *meth = _PyObject_GetAttrId(yf, &PyId_close);
if (meth == NULL) {
if (!PyErr_ExceptionMatches(PyExc_AttributeError))
PyErr_WriteUnraisable(yf);
PyErr_Clear();
} else {
retval = PyObject_CallFunction(meth, "");
Py_DECREF(meth);
if (retval == NULL)
return -1;
}
}
Py_XDECREF(retval);
return 0;
}
static PyObject *
gen_yf(PyGenObject *gen)
{
PyObject *yf = NULL;
PyFrameObject *f = gen->gi_frame;
if (f && f->f_stacktop) {
PyObject *bytecode = f->f_code->co_code;
unsigned char *code = (unsigned char *)PyBytes_AS_STRING(bytecode);
if (code[f->f_lasti + 1] != YIELD_FROM)
return NULL;
yf = f->f_stacktop[-1];
Py_INCREF(yf);
}
return yf;
if (f)
return _PyFrame_YieldingFrom(f);
else
return NULL;
}
static PyObject *
gen_close(PyGenObject *gen, PyObject *args)
{
PyObject *retval;
PyObject *yf = gen_yf(gen);
int err = 0;
PyFrameObject *f = gen->gi_frame;
if (yf) {
gen->gi_running = 1;
err = gen_close_iter(yf);
gen->gi_running = 0;
Py_DECREF(yf);
}
if (err == 0)
PyErr_SetNone(PyExc_GeneratorExit);
retval = gen_send_ex(gen, Py_None, 1);
if (retval) {
Py_DECREF(retval);
PyErr_SetString(PyExc_RuntimeError,
"generator ignored GeneratorExit");
/* For compatibility, we check gi_running before f == NULL */
if (gen->gi_running) {
PyErr_SetString(PyExc_ValueError,
"generator already executing");
return NULL;
}
if (PyErr_ExceptionMatches(PyExc_StopIteration)
|| PyErr_ExceptionMatches(PyExc_GeneratorExit)) {
PyErr_Clear(); /* ignore these errors */
Py_INCREF(Py_None);
return Py_None;
}
return NULL;
if (f == NULL)
Py_RETURN_NONE;
return _PyFrame_Finalize(f);
}
static void
gen_del(PyObject *self)
{
PyObject *res;
PyObject *error_type, *error_value, *error_traceback;
PyGenObject *gen = (PyGenObject *)self;
if (gen->gi_frame == NULL || gen->gi_frame->f_stacktop == NULL)
/* Generator isn't paused, so no need to close */
return;
/* Temporarily resurrect the object. */
assert(self->ob_refcnt == 0);
self->ob_refcnt = 1;
/* Save the current exception, if any. */
PyErr_Fetch(&error_type, &error_value, &error_traceback);
res = gen_close(gen, NULL);
if (res == NULL)
PyErr_WriteUnraisable(self);
else
Py_DECREF(res);
/* Restore the saved exception. */
PyErr_Restore(error_type, error_value, error_traceback);
/* Undo the temporary resurrection; can't use DECREF here, it would
* cause a recursive call.
*/
assert(self->ob_refcnt > 0);
if (--self->ob_refcnt == 0)
return; /* this is the normal path out */
/* close() resurrected it! Make it look like the original Py_DECREF
* never happened.
*/
{
Py_ssize_t refcnt = self->ob_refcnt;
_Py_NewReference(self);
self->ob_refcnt = refcnt;
}
assert(PyType_IS_GC(Py_TYPE(self)) &&
_Py_AS_GC(self)->gc.gc_refs != _PyGC_REFS_UNTRACKED);
/* If Py_REF_DEBUG, _Py_NewReference bumped _Py_RefTotal, so
* we need to undo that. */
_Py_DEC_REFTOTAL;
/* If Py_TRACE_REFS, _Py_NewReference re-added self to the object
* chain, so no more to do there.
* If COUNT_ALLOCS, the original decref bumped tp_frees, and
* _Py_NewReference bumped tp_allocs: both of those need to be
* undone.
*/
#ifdef COUNT_ALLOCS
--(Py_TYPE(self)->tp_frees);
--(Py_TYPE(self)->tp_allocs);
#endif
}
PyDoc_STRVAR(throw_doc,
"throw(typ[,val[,tb]]) -> raise exception in generator,\n\
return next yielded value or raise StopIteration.");
@ -306,7 +131,7 @@ gen_throw(PyGenObject *gen, PyObject *args)
int err;
if (PyErr_GivenExceptionMatches(typ, PyExc_GeneratorExit)) {
gen->gi_running = 1;
err = gen_close_iter(yf);
err = _PyFrame_CloseIterator(yf);
gen->gi_running = 0;
Py_DECREF(yf);
if (err < 0)
@ -544,7 +369,6 @@ PyTypeObject PyGen_Type = {
0, /* tp_cache */
0, /* tp_subclasses */
0, /* tp_weaklist */
gen_del, /* tp_del */
};
PyObject *
@ -556,6 +380,7 @@ PyGen_New(PyFrameObject *f)
return NULL;
}
gen->gi_frame = f;
f->f_gen = (PyObject *) gen;
Py_INCREF(f->f_code);
gen->gi_code = (PyObject *)(f->f_code);
gen->gi_running = 0;
@ -567,17 +392,5 @@ PyGen_New(PyFrameObject *f)
int
PyGen_NeedsFinalizing(PyGenObject *gen)
{
int i;
PyFrameObject *f = gen->gi_frame;
if (f == NULL || f->f_stacktop == NULL)
return 0; /* no frame or empty blockstack == no finalization */
/* Any block type besides a loop requires cleanup. */
for (i = 0; i < f->f_iblock; i++)
if (f->f_blockstack[i].b_type != SETUP_LOOP)
return 1;
/* No blocks except loops, it's safe to skip finalization. */
return 0;
}