From 49daf6dba8178c5ae5d4d65408b20566d39c36a8 Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 22 Mar 2022 12:57:19 +0000 Subject: [PATCH] bpo-47045: Remove `f_state` field (GH-31963) * Remove the f_state field from _PyInterpreterFrame * Make ownership of the frame explicit, replacing the is_generator field with an owner field. --- Include/cpython/genobject.h | 2 +- Include/cpython/pystate.h | 1 + Include/internal/pycore_frame.h | 46 ++--- .../2022-03-17-16-25-57.bpo-47045.xQgHul.rst | 3 + Modules/_xxsubinterpretersmodule.c | 5 +- Objects/frameobject.c | 129 +++++++----- Objects/genobject.c | 98 +++++----- Python/ceval.c | 185 +++++++++--------- Python/frame.c | 11 +- 9 files changed, 260 insertions(+), 220 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2022-03-17-16-25-57.bpo-47045.xQgHul.rst diff --git a/Include/cpython/genobject.h b/Include/cpython/genobject.h index b485ac6183e..40eaa19d3fa 100644 --- a/Include/cpython/genobject.h +++ b/Include/cpython/genobject.h @@ -27,7 +27,7 @@ extern "C" { char prefix##_closed; \ char prefix##_running_async; \ /* The frame */ \ - char prefix##_frame_valid; \ + int8_t prefix##_frame_state; \ PyObject *prefix##_iframe[1]; typedef struct { diff --git a/Include/cpython/pystate.h b/Include/cpython/pystate.h index 26d6f7576e5..1af21a2c947 100644 --- a/Include/cpython/pystate.h +++ b/Include/cpython/pystate.h @@ -103,6 +103,7 @@ struct _ts { This is to prevent the actual trace/profile code from being recorded in the trace/profile. */ int tracing; + int tracing_what; /* The event currently being traced, if any. */ /* Pointer to current _PyCFrame in the C stack frame of the currently, * or most recently, executing _PyEval_EvalFrameDefault. */ diff --git a/Include/internal/pycore_frame.h b/Include/internal/pycore_frame.h index e2f551ef2c0..14fba8cd1f9 100644 --- a/Include/internal/pycore_frame.h +++ b/Include/internal/pycore_frame.h @@ -5,6 +5,7 @@ extern "C" { #endif #include +#include struct _frame { PyObject_HEAD @@ -14,7 +15,6 @@ struct _frame { int f_lineno; /* Current line number. Only valid if non-zero */ char f_trace_lines; /* Emit per-line trace events? */ char f_trace_opcodes; /* Emit per-opcode trace events? */ - char f_owns_frame; /* This frame owns the frame */ /* The frame data, if this frame object owns the frame */ PyObject *_f_frame_data[1]; }; @@ -24,20 +24,19 @@ extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code); /* other API */ -/* These values are chosen so that the inline functions below all - * compare f_state to zero. - */ -enum _framestate { +typedef enum _framestate { FRAME_CREATED = -2, FRAME_SUSPENDED = -1, FRAME_EXECUTING = 0, - FRAME_RETURNED = 1, - FRAME_UNWINDING = 2, - FRAME_RAISED = 3, + FRAME_COMPLETED = 1, FRAME_CLEARED = 4 -}; +} PyFrameState; -typedef signed char PyFrameState; +enum _frameowner { + FRAME_OWNED_BY_THREAD = 0, + FRAME_OWNED_BY_GENERATOR = 1, + FRAME_OWNED_BY_FRAME_OBJECT = 2 +}; /* frame->f_lasti refers to the index of the last instruction, @@ -54,24 +53,11 @@ typedef struct _PyInterpreterFrame { struct _PyInterpreterFrame *previous; int f_lasti; /* Last instruction if called */ int stacktop; /* Offset of TOS from localsplus */ - PyFrameState f_state; /* What state the frame is in */ bool is_entry; // Whether this is the "root" frame for the current _PyCFrame. - bool is_generator; + char owner; PyObject *localsplus[1]; } _PyInterpreterFrame; -static inline int _PyFrame_IsRunnable(_PyInterpreterFrame *f) { - return f->f_state < FRAME_EXECUTING; -} - -static inline int _PyFrame_IsExecuting(_PyInterpreterFrame *f) { - return f->f_state == FRAME_EXECUTING; -} - -static inline int _PyFrameHasCompleted(_PyInterpreterFrame *f) { - return f->f_state > FRAME_EXECUTING; -} - static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) { return f->localsplus + f->f_code->co_nlocalsplus; } @@ -111,9 +97,8 @@ _PyFrame_InitializeSpecials( frame->stacktop = nlocalsplus; frame->frame_obj = NULL; frame->f_lasti = -1; - frame->f_state = FRAME_CREATED; frame->is_entry = false; - frame->is_generator = false; + frame->owner = FRAME_OWNED_BY_THREAD; } /* Gets the pointer to the locals array @@ -200,6 +185,15 @@ void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame); _PyInterpreterFrame * _PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func); + +static inline +PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame) +{ + assert(frame->owner == FRAME_OWNED_BY_GENERATOR); + size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); + return (PyGenObject *)(((char *)frame) - offset_in_gen); +} + #ifdef __cplusplus } #endif diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-03-17-16-25-57.bpo-47045.xQgHul.rst b/Misc/NEWS.d/next/Core and Builtins/2022-03-17-16-25-57.bpo-47045.xQgHul.rst new file mode 100644 index 00000000000..38825888429 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-03-17-16-25-57.bpo-47045.xQgHul.rst @@ -0,0 +1,3 @@ +Remove the ``f_state`` field from the _PyInterpreterFrame struct. Add the +``owner`` field to the _PyInterpreterFrame struct to make ownership explicit +to simplify clearing and deallocing frames and generators. diff --git a/Modules/_xxsubinterpretersmodule.c b/Modules/_xxsubinterpretersmodule.c index 846b24d5efa..0e37ce0aa3f 100644 --- a/Modules/_xxsubinterpretersmodule.c +++ b/Modules/_xxsubinterpretersmodule.c @@ -1843,10 +1843,7 @@ _is_running(PyInterpreterState *interp) if (frame == NULL) { return 0; } - - int executing = _PyFrame_IsExecuting(frame); - - return executing; + return 1; } static int diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 7ccd300e2b4..5c6a8bcb900 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -412,6 +412,42 @@ frame_stack_pop(PyFrameObject *f) Py_DECREF(v); } +static PyFrameState +_PyFrame_GetState(PyFrameObject *frame) +{ + if (frame->f_frame->stacktop == 0) { + return FRAME_CLEARED; + } + switch(frame->f_frame->owner) { + case FRAME_OWNED_BY_GENERATOR: + { + PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame); + return gen->gi_frame_state; + } + case FRAME_OWNED_BY_THREAD: + { + if (frame->f_frame->f_lasti < 0) { + return FRAME_CREATED; + } + uint8_t *code = (uint8_t *)frame->f_frame->f_code->co_code_adaptive; + int opcode = code[frame->f_frame->f_lasti*sizeof(_Py_CODEUNIT)]; + switch(_PyOpcode_Deopt[opcode]) { + case COPY_FREE_VARS: + case MAKE_CELL: + case RETURN_GENERATOR: + /* Frame not fully initialized */ + return FRAME_CREATED; + default: + return FRAME_EXECUTING; + } + } + case FRAME_OWNED_BY_FRAME_OBJECT: + return FRAME_COMPLETED; + } + Py_UNREACHABLE(); +} + + /* Setter for f_lineno - you can set f_lineno from within a trace function in * order to jump to a given line of code, subject to some restrictions. Most * lines are OK to jump to because they don't make any assumptions about the @@ -440,6 +476,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore return -1; } + PyFrameState state = _PyFrame_GetState(f); /* * This code preserves the historical restrictions on * setting the line number of a frame. @@ -448,28 +485,31 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore * In addition, jumps are forbidden when not tracing, * as this is a debugging feature. */ - switch(f->f_frame->f_state) { - case FRAME_CREATED: - PyErr_Format(PyExc_ValueError, - "can't jump from the 'call' trace event of a new frame"); - return -1; - case FRAME_RETURNED: - case FRAME_UNWINDING: - case FRAME_RAISED: - case FRAME_CLEARED: + switch(PyThreadState_GET()->tracing_what) { + case PyTrace_EXCEPTION: PyErr_SetString(PyExc_ValueError, "can only jump from a 'line' trace event"); return -1; - case FRAME_EXECUTING: - case FRAME_SUSPENDED: - /* You can only do this from within a trace function, not via - * _getframe or similar hackery. */ - if (!f->f_trace) { - PyErr_Format(PyExc_ValueError, - "f_lineno can only be set by a trace function"); - return -1; - } + case PyTrace_CALL: + PyErr_Format(PyExc_ValueError, + "can't jump from the 'call' trace event of a new frame"); + return -1; + case PyTrace_LINE: break; + case PyTrace_RETURN: + if (state == FRAME_SUSPENDED) { + break; + } + /* fall through */ + default: + PyErr_SetString(PyExc_ValueError, + "can only jump from a 'line' trace event"); + return -1; + } + if (!f->f_trace) { + PyErr_Format(PyExc_ValueError, + "f_lineno can only be set by a trace function"); + return -1; } int new_lineno; @@ -555,8 +595,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore PyErr_SetString(PyExc_ValueError, msg); return -1; } - /* Unwind block stack. */ - if (f->f_frame->f_state == FRAME_SUSPENDED) { + if (state == FRAME_SUSPENDED) { /* Account for value popped by yield */ start_stack = pop_value(start_stack); } @@ -623,7 +662,9 @@ frame_dealloc(PyFrameObject *f) { /* It is the responsibility of the owning generator/coroutine * to have cleared the generator pointer */ - assert(!f->f_frame->is_generator); + + assert(f->f_frame->owner != FRAME_OWNED_BY_GENERATOR || + _PyFrame_GetGenerator(f->f_frame)->gi_frame_state == FRAME_CLEARED); if (_PyObject_GC_IS_TRACKED(f)) { _PyObject_GC_UNTRACK(f); @@ -633,8 +674,7 @@ frame_dealloc(PyFrameObject *f) PyCodeObject *co = NULL; /* Kill all local variables including specials, if we own them */ - if (f->f_owns_frame) { - f->f_owns_frame = 0; + if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) { assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data; /* Don't clear code object until the end */ @@ -659,7 +699,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) { Py_VISIT(f->f_back); Py_VISIT(f->f_trace); - if (f->f_owns_frame == 0) { + if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) { return 0; } assert(f->f_frame->frame_obj == NULL); @@ -669,13 +709,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg) static int frame_tp_clear(PyFrameObject *f) { - /* Before anything else, make sure that this frame is clearly marked - * as being defunct! Else, e.g., a generator reachable from this - * frame may also point to this frame, believe itself to still be - * active, and try cleaning up this frame again. - */ - f->f_frame->f_state = FRAME_CLEARED; - Py_CLEAR(f->f_trace); /* locals and stack */ @@ -691,19 +724,25 @@ frame_tp_clear(PyFrameObject *f) static PyObject * frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored)) { - if (_PyFrame_IsExecuting(f->f_frame)) { - PyErr_SetString(PyExc_RuntimeError, - "cannot clear an executing frame"); - return NULL; + if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) { + PyGenObject *gen = _PyFrame_GetGenerator(f->f_frame); + if (gen->gi_frame_state == FRAME_EXECUTING) { + goto running; + } + _PyGen_Finalize((PyObject *)gen); } - if (f->f_frame->is_generator) { - assert(!f->f_owns_frame); - size_t offset_in_gen = offsetof(PyGenObject, gi_iframe); - PyObject *gen = (PyObject *)(((char *)f->f_frame) - offset_in_gen); - _PyGen_Finalize(gen); + else if (f->f_frame->owner == FRAME_OWNED_BY_THREAD) { + goto running; + } + else { + assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); + (void)frame_tp_clear(f); } - (void)frame_tp_clear(f); Py_RETURN_NONE; +running: + PyErr_SetString(PyExc_RuntimeError, + "cannot clear an executing frame"); + return NULL; } PyDoc_STRVAR(clear__doc__, @@ -835,7 +874,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code, } init_frame((_PyInterpreterFrame *)f->_f_frame_data, func, locals); f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data; - f->f_owns_frame = 1; + f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; Py_DECREF(func); _PyObject_GC_TRACK(f); return f; @@ -912,7 +951,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) { PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i); PyObject *value = fast[i]; - if (frame->f_state != FRAME_CLEARED) { + if (frame->stacktop) { if (kind & CO_FAST_FREE) { // The cell was set by COPY_FREE_VARS. assert(value != NULL && PyCell_Check(value)); @@ -1049,7 +1088,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear) void PyFrame_LocalsToFast(PyFrameObject *f, int clear) { - if (f == NULL || f->f_frame->f_state == FRAME_CLEARED) { + if (f == NULL || _PyFrame_GetState(f) == FRAME_CLEARED) { return; } _PyFrame_LocalsToFast(f->f_frame, clear); @@ -1096,3 +1135,5 @@ _PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals) return _PyEval_GetBuiltins(tstate); } + + diff --git a/Objects/genobject.c b/Objects/genobject.c index 3ad8dc1c459..f071390d6d3 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -35,9 +35,10 @@ gen_traverse(PyGenObject *gen, visitproc visit, void *arg) Py_VISIT(gen->gi_code); Py_VISIT(gen->gi_name); Py_VISIT(gen->gi_qualname); - if (gen->gi_frame_valid) { + if (gen->gi_frame_state < FRAME_CLEARED) { _PyInterpreterFrame *frame = (_PyInterpreterFrame *)(gen->gi_iframe); - assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0); + assert(frame->frame_obj == NULL || + frame->frame_obj->f_frame->owner == FRAME_OWNED_BY_GENERATOR); int err = _PyFrame_Traverse(frame, visit, arg); if (err) { return err; @@ -55,7 +56,7 @@ _PyGen_Finalize(PyObject *self) PyObject *res = NULL; PyObject *error_type, *error_value, *error_traceback; - if (gen->gi_frame_valid == 0 || _PyFrameHasCompleted((_PyInterpreterFrame *)gen->gi_iframe)) { + if (gen->gi_frame_state >= FRAME_COMPLETED) { /* Generator isn't paused, so no need to close */ return; } @@ -87,7 +88,7 @@ _PyGen_Finalize(PyObject *self) issue a RuntimeWarning. */ if (gen->gi_code != NULL && ((PyCodeObject *)gen->gi_code)->co_flags & CO_COROUTINE && - ((_PyInterpreterFrame *)gen->gi_iframe)->f_state == FRAME_CREATED) + gen->gi_frame_state == FRAME_CREATED) { _PyErr_WarnUnawaitedCoroutine((PyObject *)gen); } @@ -130,10 +131,9 @@ gen_dealloc(PyGenObject *gen) and GC_Del. */ Py_CLEAR(((PyAsyncGenObject*)gen)->ag_origin_or_finalizer); } - if (gen->gi_frame_valid) { + if (gen->gi_frame_state < FRAME_CLEARED) { _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - gen->gi_frame_valid = 0; - frame->is_generator = false; + gen->gi_frame_state = FRAME_CLEARED; frame->previous = NULL; _PyFrame_Clear(frame); } @@ -156,7 +156,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, PyObject *result; *presult = NULL; - if (frame->f_state == FRAME_CREATED && arg && arg != Py_None) { + if (gen->gi_frame_state == FRAME_CREATED && arg && arg != Py_None) { const char *msg = "can't send non-None value to a " "just-started generator"; if (PyCoro_CheckExact(gen)) { @@ -169,7 +169,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, PyErr_SetString(PyExc_TypeError, msg); return PYGEN_ERROR; } - if (gen->gi_frame_valid && _PyFrame_IsExecuting(frame)) { + if (gen->gi_frame_state == FRAME_EXECUTING) { const char *msg = "generator already executing"; if (PyCoro_CheckExact(gen)) { msg = "coroutine already executing"; @@ -180,7 +180,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, PyErr_SetString(PyExc_ValueError, msg); return PYGEN_ERROR; } - if (gen->gi_frame_valid == 0 || _PyFrameHasCompleted(frame)) { + if (gen->gi_frame_state >= FRAME_COMPLETED) { if (PyCoro_CheckExact(gen) && !closing) { /* `gen` is an exhausted coroutine: raise an error, except when called from gen_close(), which should @@ -199,8 +199,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, return PYGEN_ERROR; } - assert(gen->gi_frame_valid); - assert(_PyFrame_IsRunnable(frame)); + assert(gen->gi_frame_state < FRAME_EXECUTING); /* Push arg onto the frame's value stack */ result = arg ? arg : Py_None; Py_INCREF(result); @@ -216,7 +215,11 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, _PyErr_ChainStackItem(NULL); } + gen->gi_frame_state = FRAME_EXECUTING; result = _PyEval_EvalFrame(tstate, frame, exc); + if (gen->gi_frame_state == FRAME_EXECUTING) { + gen->gi_frame_state = FRAME_COMPLETED; + } tstate->exc_info = gen->gi_exc_state.previous_item; gen->gi_exc_state.previous_item = NULL; @@ -229,7 +232,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* If the generator just returned (as opposed to yielding), signal * that the generator is exhausted. */ if (result) { - if (!_PyFrameHasCompleted(frame)) { + if (gen->gi_frame_state == FRAME_SUSPENDED) { *presult = result; return PYGEN_NEXT; } @@ -265,8 +268,7 @@ gen_send_ex2(PyGenObject *gen, PyObject *arg, PyObject **presult, /* first clean reference cycle through stored exception traceback */ _PyErr_ClearExcState(&gen->gi_exc_state); - frame->is_generator = false; - gen->gi_frame_valid = 0; + gen->gi_frame_state = FRAME_CLEARED; _PyFrame_Clear(frame); *presult = result; return result ? PYGEN_RETURN : PYGEN_ERROR; @@ -347,7 +349,7 @@ _PyGen_yf(PyGenObject *gen) { PyObject *yf = NULL; - if (gen->gi_frame_valid) { + if (gen->gi_frame_state < FRAME_CLEARED) { _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; if (frame->f_lasti < 1) { @@ -378,11 +380,10 @@ gen_close(PyGenObject *gen, PyObject *args) int err = 0; if (yf) { - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; - PyFrameState state = frame->f_state; - frame->f_state = FRAME_EXECUTING; + PyFrameState state = gen->gi_frame_state; + gen->gi_frame_state = FRAME_EXECUTING; err = gen_close_iter(yf); - frame->f_state = state; + gen->gi_frame_state = state; Py_DECREF(yf); } if (err == 0) @@ -429,10 +430,10 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, We have to allow some awaits to work it through, hence the `close_on_genexit` parameter here. */ - PyFrameState state = frame->f_state; - frame->f_state = FRAME_EXECUTING; + PyFrameState state = gen->gi_frame_state; + gen->gi_frame_state = FRAME_EXECUTING; err = gen_close_iter(yf); - frame->f_state = state; + gen->gi_frame_state = state; Py_DECREF(yf); if (err < 0) return gen_send_ex(gen, Py_None, 1, 0); @@ -451,11 +452,11 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, tstate->cframe->current_frame = frame; /* Close the generator that we are currently iterating with 'yield from' or awaiting on with 'await'. */ - PyFrameState state = frame->f_state; - frame->f_state = FRAME_EXECUTING; + PyFrameState state = gen->gi_frame_state; + gen->gi_frame_state = FRAME_EXECUTING; ret = _gen_throw((PyGenObject *)yf, close_on_genexit, typ, val, tb); - frame->f_state = state; + gen->gi_frame_state = state; tstate->cframe->current_frame = prev; frame->previous = NULL; } else { @@ -469,17 +470,17 @@ _gen_throw(PyGenObject *gen, int close_on_genexit, Py_DECREF(yf); goto throw_here; } - PyFrameState state = frame->f_state; - frame->f_state = FRAME_EXECUTING; + PyFrameState state = gen->gi_frame_state; + gen->gi_frame_state = FRAME_EXECUTING; ret = PyObject_CallFunctionObjArgs(meth, typ, val, tb, NULL); - frame->f_state = state; + gen->gi_frame_state = state; Py_DECREF(meth); } Py_DECREF(yf); if (!ret) { PyObject *val; /* Pop subiterator from stack */ - assert(gen->gi_frame_valid); + assert(gen->gi_frame_state < FRAME_CLEARED); ret = _PyFrame_StackPop((_PyInterpreterFrame *)gen->gi_iframe); assert(ret == yf); Py_DECREF(ret); @@ -756,19 +757,16 @@ gen_getyieldfrom(PyGenObject *gen, void *Py_UNUSED(ignored)) static PyObject * gen_getrunning(PyGenObject *gen, void *Py_UNUSED(ignored)) { - if (gen->gi_frame_valid == 0) { - Py_RETURN_FALSE; + if (gen->gi_frame_state == FRAME_EXECUTING) { + Py_RETURN_TRUE; } - return PyBool_FromLong(_PyFrame_IsExecuting((_PyInterpreterFrame *)gen->gi_iframe)); + Py_RETURN_FALSE; } static PyObject * gen_getsuspended(PyGenObject *gen, void *Py_UNUSED(ignored)) { - if (gen->gi_frame_valid == 0) { - Py_RETURN_FALSE; - } - return PyBool_FromLong(((_PyInterpreterFrame *)gen->gi_iframe)->f_state == FRAME_SUSPENDED); + return PyBool_FromLong(gen->gi_frame_state == FRAME_SUSPENDED); } static PyObject * @@ -777,7 +775,7 @@ _gen_getframe(PyGenObject *gen, const char *const name) if (PySys_Audit("object.__getattr__", "Os", gen, name) < 0) { return NULL; } - if (gen->gi_frame_valid == 0) { + if (gen->gi_frame_state == FRAME_CLEARED) { Py_RETURN_NONE; } return _Py_XNewRef((PyObject *)_PyFrame_GetFrameObject((_PyInterpreterFrame *)gen->gi_iframe)); @@ -899,7 +897,7 @@ make_gen(PyTypeObject *type, PyFunctionObject *func) if (gen == NULL) { return NULL; } - gen->gi_frame_valid = 0; + gen->gi_frame_state = FRAME_CLEARED; gen->gi_code = (PyCodeObject *)func->func_code; Py_INCREF(gen->gi_code); gen->gi_weakreflist = NULL; @@ -972,14 +970,13 @@ gen_new_with_qualname(PyTypeObject *type, PyFrameObject *f, } /* Copy the frame */ assert(f->f_frame->frame_obj == NULL); - assert(f->f_owns_frame); + assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT); _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; _PyFrame_Copy((_PyInterpreterFrame *)f->_f_frame_data, frame); - gen->gi_frame_valid = 1; + gen->gi_frame_state = FRAME_CREATED; assert(frame->frame_obj == f); - f->f_owns_frame = 0; f->f_frame = frame; - frame->is_generator = true; + frame->owner = FRAME_OWNED_BY_GENERATOR; assert(PyObject_GC_IsTracked((PyObject *)f)); gen->gi_code = PyFrame_GetCode(f); Py_INCREF(gen->gi_code); @@ -1114,19 +1111,19 @@ coro_get_cr_await(PyCoroObject *coro, void *Py_UNUSED(ignored)) static PyObject * cr_getsuspended(PyCoroObject *coro, void *Py_UNUSED(ignored)) { - if (coro->cr_frame_valid == 0) { - Py_RETURN_FALSE; + if (coro->cr_frame_state == FRAME_SUSPENDED) { + Py_RETURN_TRUE; } - return PyBool_FromLong(((_PyInterpreterFrame *)coro->cr_iframe)->f_state == FRAME_SUSPENDED); + Py_RETURN_FALSE; } static PyObject * cr_getrunning(PyCoroObject *coro, void *Py_UNUSED(ignored)) { - if (coro->cr_frame_valid == 0) { - Py_RETURN_FALSE; + if (coro->cr_frame_state == FRAME_EXECUTING) { + Py_RETURN_TRUE; } - return PyBool_FromLong(_PyFrame_IsExecuting((_PyInterpreterFrame *)coro->cr_iframe)); + Py_RETURN_FALSE; } static PyObject * @@ -2063,7 +2060,6 @@ static PyObject * async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) { PyGenObject *gen = (PyGenObject*)o->agt_gen; - _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; PyObject *retval; if (o->agt_state == AWAITABLE_STATE_CLOSED) { @@ -2073,7 +2069,7 @@ async_gen_athrow_send(PyAsyncGenAThrow *o, PyObject *arg) return NULL; } - if (gen->gi_frame_valid == 0 || _PyFrameHasCompleted(frame)) { + if (gen->gi_frame_state >= FRAME_COMPLETED) { o->agt_state = AWAITABLE_STATE_CLOSED; PyErr_SetNone(PyExc_StopIteration); return NULL; diff --git a/Python/ceval.c b/Python/ceval.c index 7c6bfd46900..73179c810b7 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1730,7 +1730,6 @@ handle_eval_breaker: PREDICTED(RESUME_QUICK); assert(tstate->cframe == &cframe); assert(frame == cframe.current_frame); - frame->f_state = FRAME_EXECUTING; if (_Py_atomic_load_relaxed(eval_breaker) && oparg < 2) { goto handle_eval_breaker; } @@ -2373,7 +2372,6 @@ handle_eval_breaker: TARGET(RETURN_VALUE) { PyObject *retval = POP(); assert(EMPTY()); - frame->f_state = FRAME_RETURNED; _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); @@ -2585,7 +2583,7 @@ handle_eval_breaker: TARGET(YIELD_VALUE) { assert(frame->is_entry); PyObject *retval = POP(); - frame->f_state = FRAME_SUSPENDED; + _PyFrame_GetGenerator(frame)->gi_frame_state = FRAME_SUSPENDED; _PyFrame_SetStackPointer(frame, stack_pointer); TRACE_FUNCTION_EXIT(); DTRACE_FUNCTION_EXIT(); @@ -4068,7 +4066,6 @@ handle_eval_breaker: * generator or coroutine, so we deliberately do not check it here. * (see bpo-30039). */ - frame->f_state = FRAME_EXECUTING; JUMPTO(oparg); DISPATCH(); } @@ -5253,9 +5250,8 @@ handle_eval_breaker: _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; _PyFrame_Copy(frame, gen_frame); assert(frame->frame_obj == NULL); - gen->gi_frame_valid = 1; - gen_frame->is_generator = true; - gen_frame->f_state = FRAME_CREATED; + gen->gi_frame_state = FRAME_CREATED; + gen_frame->owner = FRAME_OWNED_BY_GENERATOR; _Py_LeaveRecursiveCall(tstate); if (!frame->is_entry) { _PyInterpreterFrame *prev = frame->previous; @@ -5429,41 +5425,47 @@ handle_eval_breaker: int instr_prev = frame->f_lasti; frame->f_lasti = INSTR_OFFSET(); TRACING_NEXTOPARG(); - if (opcode == RESUME) { - if (oparg < 2) { - CHECK_EVAL_BREAKER(); - } - /* Call tracing */ - TRACE_FUNCTION_ENTRY(); - DTRACE_FUNCTION_ENTRY(); - } - else if (frame->f_state > FRAME_CREATED) { - /* line-by-line tracing support */ - if (PyDTrace_LINE_ENABLED()) { - maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); - } - - if (cframe.use_tracing && - tstate->c_tracefunc != NULL && !tstate->tracing) { - int err; - /* see maybe_call_line_trace() - for expository comments */ - _PyFrame_SetStackPointer(frame, stack_pointer); - - err = maybe_call_line_trace(tstate->c_tracefunc, - tstate->c_traceobj, - tstate, frame, instr_prev); - if (err) { - /* trace function raised an exception */ - next_instr++; - goto error; + switch(opcode) { + case COPY_FREE_VARS: + case MAKE_CELL: + case RETURN_GENERATOR: + /* Frame not fully initialized */ + break; + case RESUME: + if (oparg < 2) { + CHECK_EVAL_BREAKER(); + } + /* Call tracing */ + TRACE_FUNCTION_ENTRY(); + DTRACE_FUNCTION_ENTRY(); + break; + default: + /* line-by-line tracing support */ + if (PyDTrace_LINE_ENABLED()) { + maybe_dtrace_line(frame, &tstate->trace_info, instr_prev); } - /* Reload possibly changed frame fields */ - JUMPTO(frame->f_lasti); - stack_pointer = _PyFrame_GetStackPointer(frame); - frame->stacktop = -1; - } + if (cframe.use_tracing && + tstate->c_tracefunc != NULL && !tstate->tracing) { + int err; + /* see maybe_call_line_trace() + for expository comments */ + _PyFrame_SetStackPointer(frame, stack_pointer); + + err = maybe_call_line_trace(tstate->c_tracefunc, + tstate->c_traceobj, + tstate, frame, instr_prev); + if (err) { + /* trace function raised an exception */ + next_instr++; + goto error; + } + /* Reload possibly changed frame fields */ + JUMPTO(frame->f_lasti); + + stack_pointer = _PyFrame_GetStackPointer(frame); + frame->stacktop = -1; + } } } TRACING_NEXTOPARG(); @@ -5558,65 +5560,63 @@ error: if (tstate->c_tracefunc != NULL) { /* Make sure state is set to FRAME_UNWINDING for tracing */ - frame->f_state = FRAME_UNWINDING; call_exc_trace(tstate->c_tracefunc, tstate->c_traceobj, tstate, frame); } exception_unwind: - frame->f_state = FRAME_UNWINDING; - /* We can't use frame->f_lasti here, as RERAISE may have set it */ - int offset = INSTR_OFFSET()-1; - int level, handler, lasti; - if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) { - // No handlers, so exit. - assert(_PyErr_Occurred(tstate)); + { + /* We can't use frame->f_lasti here, as RERAISE may have set it */ + int offset = INSTR_OFFSET()-1; + int level, handler, lasti; + if (get_exception_handler(frame->f_code, offset, &level, &handler, &lasti) == 0) { + // No handlers, so exit. + assert(_PyErr_Occurred(tstate)); - /* Pop remaining stack entries. */ - PyObject **stackbase = _PyFrame_Stackbase(frame); - while (stack_pointer > stackbase) { - PyObject *o = POP(); - Py_XDECREF(o); + /* Pop remaining stack entries. */ + PyObject **stackbase = _PyFrame_Stackbase(frame); + while (stack_pointer > stackbase) { + PyObject *o = POP(); + Py_XDECREF(o); + } + assert(STACK_LEVEL() == 0); + _PyFrame_SetStackPointer(frame, stack_pointer); + TRACE_FUNCTION_UNWIND(); + DTRACE_FUNCTION_EXIT(); + goto exit_unwind; } - assert(STACK_LEVEL() == 0); - _PyFrame_SetStackPointer(frame, stack_pointer); - frame->f_state = FRAME_RAISED; - TRACE_FUNCTION_UNWIND(); - DTRACE_FUNCTION_EXIT(); - goto exit_unwind; - } - assert(STACK_LEVEL() >= level); - PyObject **new_top = _PyFrame_Stackbase(frame) + level; - while (stack_pointer > new_top) { - PyObject *v = POP(); - Py_XDECREF(v); - } - PyObject *exc, *val, *tb; - if (lasti) { - PyObject *lasti = PyLong_FromLong(frame->f_lasti); - if (lasti == NULL) { - goto exception_unwind; + assert(STACK_LEVEL() >= level); + PyObject **new_top = _PyFrame_Stackbase(frame) + level; + while (stack_pointer > new_top) { + PyObject *v = POP(); + Py_XDECREF(v); } - PUSH(lasti); + PyObject *exc, *val, *tb; + if (lasti) { + PyObject *lasti = PyLong_FromLong(frame->f_lasti); + if (lasti == NULL) { + goto exception_unwind; + } + PUSH(lasti); + } + _PyErr_Fetch(tstate, &exc, &val, &tb); + /* Make the raw exception data + available to the handler, + so a program can emulate the + Python main loop. */ + _PyErr_NormalizeException(tstate, &exc, &val, &tb); + if (tb != NULL) + PyException_SetTraceback(val, tb); + else + PyException_SetTraceback(val, Py_None); + Py_XDECREF(tb); + Py_XDECREF(exc); + PUSH(val); + JUMPTO(handler); + /* Resume normal execution */ + DISPATCH(); } - _PyErr_Fetch(tstate, &exc, &val, &tb); - /* Make the raw exception data - available to the handler, - so a program can emulate the - Python main loop. */ - _PyErr_NormalizeException(tstate, &exc, &val, &tb); - if (tb != NULL) - PyException_SetTraceback(val, tb); - else - PyException_SetTraceback(val, Py_None); - Py_XDECREF(tb); - Py_XDECREF(exc); - PUSH(val); - JUMPTO(handler); - /* Resume normal execution */ - frame->f_state = FRAME_EXECUTING; - DISPATCH(); } exit_unwind: @@ -6180,6 +6180,7 @@ _PyEvalFramePushAndInit(PyThreadState *tstate, PyFunctionObject *func, localsarray[i] = NULL; } if (initialize_locals(tstate, func, localsarray, args, argcount, kwnames)) { + assert(frame->owner != FRAME_OWNED_BY_GENERATOR); _PyFrame_Clear(frame); return NULL; } @@ -6203,7 +6204,8 @@ static void _PyEvalFrameClearAndPop(PyThreadState *tstate, _PyInterpreterFrame * frame) { tstate->recursion_remaining--; - assert(frame->frame_obj == NULL || frame->frame_obj->f_owns_frame == 0); + assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); + assert(frame->owner == FRAME_OWNED_BY_THREAD); _PyFrame_Clear(frame); tstate->recursion_remaining++; _PyThreadState_PopFrame(tstate, frame); @@ -6681,6 +6683,8 @@ call_trace(Py_tracefunc func, PyObject *obj, if (f == NULL) { return -1; } + int old_what = tstate->tracing_what; + tstate->tracing_what = what; PyThreadState_EnterTracing(tstate); assert (frame->f_lasti >= 0); initialize_trace_info(&tstate->trace_info, frame); @@ -6688,6 +6692,7 @@ call_trace(Py_tracefunc func, PyObject *obj, result = func(obj, f, what, arg); f->f_lineno = 0; PyThreadState_LeaveTracing(tstate); + tstate->tracing_what = old_what; return result; } diff --git a/Python/frame.c b/Python/frame.c index 20b4f81425b..3396ed8d2ae 100644 --- a/Python/frame.c +++ b/Python/frame.c @@ -37,7 +37,8 @@ _PyFrame_MakeAndSetFrameObject(_PyInterpreterFrame *frame) Py_XDECREF(error_traceback); } else { - f->f_owns_frame = 0; + assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); + assert(frame->owner != FRAME_CLEARED); f->f_frame = frame; frame->frame_obj = f; PyErr_Restore(error_type, error_value, error_traceback); @@ -57,12 +58,13 @@ _PyFrame_Copy(_PyInterpreterFrame *src, _PyInterpreterFrame *dest) static void take_ownership(PyFrameObject *f, _PyInterpreterFrame *frame) { - assert(f->f_owns_frame == 0); + assert(frame->owner != FRAME_OWNED_BY_FRAME_OBJECT); + assert(frame->owner != FRAME_CLEARED); Py_ssize_t size = ((char*)&frame->localsplus[frame->stacktop]) - (char *)frame; memcpy((_PyInterpreterFrame *)f->_f_frame_data, frame, size); frame = (_PyInterpreterFrame *)f->_f_frame_data; - f->f_owns_frame = 1; f->f_frame = frame; + frame->owner = FRAME_OWNED_BY_FRAME_OBJECT; assert(f->f_back == NULL); if (frame->previous != NULL) { /* Link PyFrameObjects.f_back and remove link through _PyInterpreterFrame.previous */ @@ -88,7 +90,8 @@ _PyFrame_Clear(_PyInterpreterFrame *frame) { /* It is the responsibility of the owning generator/coroutine * to have cleared the enclosing generator, if any. */ - assert(!frame->is_generator); + assert(frame->owner != FRAME_OWNED_BY_GENERATOR || + _PyFrame_GetGenerator(frame)->gi_frame_state == FRAME_CLEARED); if (frame->frame_obj) { PyFrameObject *f = frame->frame_obj; frame->frame_obj = NULL;