diff --git a/Include/internal/pycore_floatobject.h b/Include/internal/pycore_floatobject.h index a099f2ebc0f..8a655543329 100644 --- a/Include/internal/pycore_floatobject.h +++ b/Include/internal/pycore_floatobject.h @@ -38,6 +38,8 @@ struct _Py_float_state { #endif }; +void _PyFloat_ExactDealloc(PyObject *op); + PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); diff --git a/Include/internal/pycore_object.h b/Include/internal/pycore_object.h index 177b06e2dd4..f022f824698 100644 --- a/Include/internal/pycore_object.h +++ b/Include/internal/pycore_object.h @@ -14,7 +14,6 @@ extern "C" { #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_runtime.h" // _PyRuntime - #define _PyObject_IMMORTAL_INIT(type) \ { \ .ob_refcnt = 999999999, \ @@ -26,6 +25,42 @@ extern "C" { .ob_size = size, \ } +PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( + const char *func, + const char *message); + +#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message) + +static inline void +_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct) +{ +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + if (--op->ob_refcnt != 0) { + assert(op->ob_refcnt > 0); + } + else { +#ifdef Py_TRACE_REFS + _Py_ForgetReference(op); +#endif + destruct(op); + } +} + +static inline void +_Py_DECREF_NO_DEALLOC(PyObject *op) +{ +#ifdef Py_REF_DEBUG + _Py_RefTotal--; +#endif + op->ob_refcnt--; +#ifdef Py_DEBUG + if (op->ob_refcnt <= 0) { + _Py_FatalRefcountError("Expected a positive remaining refcount"); + } +#endif +} PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type); PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content); diff --git a/Include/internal/pycore_pyerrors.h b/Include/internal/pycore_pyerrors.h index e3c445ba5d9..66f37942ef9 100644 --- a/Include/internal/pycore_pyerrors.h +++ b/Include/internal/pycore_pyerrors.h @@ -100,13 +100,6 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception); PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b, Py_ssize_t max_cost); -PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc( - const char *func, - const char *message); - -#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message) - - #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index 75b90501db1..4bee2419fbd 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -10,6 +10,7 @@ extern "C" { #include "pycore_fileutils.h" // _Py_error_handler +void _PyUnicode_ExactDealloc(PyObject *op); /* runtime lifecycle */ diff --git a/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst new file mode 100644 index 00000000000..e19ce0a243c --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst @@ -0,0 +1 @@ +Add type-specialized versions of the ``Py_DECREF()``, and use them for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime where types are known at C compile time. diff --git a/Objects/boolobject.c b/Objects/boolobject.c index d86958aff9c..ff7218760ab 100644 --- a/Objects/boolobject.c +++ b/Objects/boolobject.c @@ -1,8 +1,8 @@ /* Boolean type, a subtype of int */ #include "Python.h" +#include "pycore_object.h" // _Py_FatalRefcountError() #include "pycore_runtime.h" // _Py_ID() -#include "pycore_pyerrors.h" // _Py_FatalRefcountError() /* We define bool_repr to return "False" or "True" */ diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 736ddc95d68..a5774b9e300 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -238,28 +238,41 @@ PyFloat_FromString(PyObject *v) return result; } -static void -float_dealloc(PyFloatObject *op) +void +_PyFloat_ExactDealloc(PyObject *obj) { + assert(PyFloat_CheckExact(obj)); + PyFloatObject *op = (PyFloatObject *)obj; +#if PyFloat_MAXFREELIST > 0 + struct _Py_float_state *state = get_float_state(); +#ifdef Py_DEBUG + // float_dealloc() must not be called after _PyFloat_Fini() + assert(state->numfree != -1); +#endif + if (state->numfree >= PyFloat_MAXFREELIST) { + PyObject_Free(op); + return; + } + state->numfree++; + Py_SET_TYPE(op, (PyTypeObject *)state->free_list); + state->free_list = op; +#else + PyObject_Free(op); +#endif +} + +static void +float_dealloc(PyObject *op) +{ + assert(PyFloat_Check(op)); #if PyFloat_MAXFREELIST > 0 if (PyFloat_CheckExact(op)) { - struct _Py_float_state *state = get_float_state(); -#ifdef Py_DEBUG - // float_dealloc() must not be called after _PyFloat_Fini() - assert(state->numfree != -1); -#endif - if (state->numfree >= PyFloat_MAXFREELIST) { - PyObject_Free(op); - return; - } - state->numfree++; - Py_SET_TYPE(op, (PyTypeObject *)state->free_list); - state->free_list = op; + _PyFloat_ExactDealloc(op); } else #endif { - Py_TYPE(op)->tp_free((PyObject *)op); + Py_TYPE(op)->tp_free(op); } } diff --git a/Objects/longobject.c b/Objects/longobject.c index cc4aef31d74..c104dcc14f9 100644 --- a/Objects/longobject.c +++ b/Objects/longobject.c @@ -36,7 +36,15 @@ medium_value(PyLongObject *x) #define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS) #define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS) -static inline int is_medium_int(stwodigits x) +static inline void +_Py_DECREF_INT(PyLongObject *op) +{ + assert(PyLong_CheckExact(op)); + _Py_DECREF_SPECIALIZED((PyObject *)op, PyObject_Free); +} + +static inline int +is_medium_int(stwodigits x) { /* Take care that we are comparing unsigned values. */ twodigits x_plus_mask = ((twodigits)x) + PyLong_MASK; @@ -58,7 +66,7 @@ maybe_small_long(PyLongObject *v) if (v && IS_MEDIUM_VALUE(v)) { stwodigits ival = medium_value(v); if (IS_SMALL_INT(ival)) { - Py_DECREF(v); + _Py_DECREF_INT(v); return (PyLongObject *)get_small_int((sdigit)ival); } } @@ -1856,7 +1864,7 @@ long_to_decimal_string_internal(PyObject *aa, #undef WRITE_DIGITS #undef WRITE_UNICODE_DIGITS - Py_DECREF(scratch); + _Py_DECREF_INT(scratch); if (writer) { writer->pos += strlen; } @@ -3561,15 +3569,15 @@ k_mul(PyLongObject *a, PyLongObject *b) */ i = Py_SIZE(ret) - shift; /* # digits after shift */ (void)v_isub(ret->ob_digit + shift, i, t2->ob_digit, Py_SIZE(t2)); - Py_DECREF(t2); + _Py_DECREF_INT(t2); (void)v_isub(ret->ob_digit + shift, i, t1->ob_digit, Py_SIZE(t1)); - Py_DECREF(t1); + _Py_DECREF_INT(t1); /* 6. t3 <- (ah+al)(bh+bl), and add into result. */ if ((t1 = x_add(ah, al)) == NULL) goto fail; - Py_DECREF(ah); - Py_DECREF(al); + _Py_DECREF_INT(ah); + _Py_DECREF_INT(al); ah = al = NULL; if (a == b) { @@ -3580,13 +3588,13 @@ k_mul(PyLongObject *a, PyLongObject *b) Py_DECREF(t1); goto fail; } - Py_DECREF(bh); - Py_DECREF(bl); + _Py_DECREF_INT(bh); + _Py_DECREF_INT(bl); bh = bl = NULL; t3 = k_mul(t1, t2); - Py_DECREF(t1); - Py_DECREF(t2); + _Py_DECREF_INT(t1); + _Py_DECREF_INT(t2); if (t3 == NULL) goto fail; assert(Py_SIZE(t3) >= 0); @@ -3594,7 +3602,7 @@ k_mul(PyLongObject *a, PyLongObject *b) * See the (*) comment after this function. */ (void)v_iadd(ret->ob_digit + shift, i, t3->ob_digit, Py_SIZE(t3)); - Py_DECREF(t3); + _Py_DECREF_INT(t3); return long_normalize(ret); @@ -3699,13 +3707,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b) /* Add into result. */ (void)v_iadd(ret->ob_digit + nbdone, Py_SIZE(ret) - nbdone, product->ob_digit, Py_SIZE(product)); - Py_DECREF(product); + _Py_DECREF_INT(product); bsize -= nbtouse; nbdone += nbtouse; } - Py_DECREF(bslice); + _Py_DECREF_INT(bslice); return long_normalize(ret); fail: @@ -5993,7 +6001,7 @@ PyTypeObject PyLong_Type = { 0, /* tp_init */ 0, /* tp_alloc */ long_new, /* tp_new */ - PyObject_Del, /* tp_free */ + PyObject_Free, /* tp_free */ }; static PyTypeObject Int_InfoType; diff --git a/Objects/object.c b/Objects/object.c index fe2d76f578e..8adb5065c2a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -9,7 +9,7 @@ #include "pycore_floatobject.h" // _PyFloat_DebugMallocStats() #include "pycore_initconfig.h" // _PyStatus_EXCEPTION() #include "pycore_namespace.h" // _PyNamespace_Type -#include "pycore_object.h" // _PyType_CheckConsistency() +#include "pycore_object.h" // _PyType_CheckConsistency(), _Py_FatalRefcountError() #include "pycore_pyerrors.h" // _PyErr_Occurred() #include "pycore_pymem.h" // _PyMem_IsPtrFreed() #include "pycore_pystate.h" // _PyThreadState_GET() diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index 306a9112b06..f21d4da2459 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -5,8 +5,7 @@ #include "pycore_abstract.h" // _PyIndex_Check() #include "pycore_gc.h" // _PyObject_GC_IS_TRACKED() #include "pycore_initconfig.h" // _PyStatus_OK() -#include "pycore_object.h" // _PyObject_GC_TRACK() -#include "pycore_pyerrors.h" // _Py_FatalRefcountError() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() /*[clinic input] class tuple "PyTupleObject *" "&PyTuple_Type" diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index dd0c0221d93..7768f66c956 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -48,9 +48,8 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. #include "pycore_initconfig.h" // _PyStatus_OK() #include "pycore_interp.h" // PyInterpreterState.fs_codec #include "pycore_long.h" // _PyLong_FormatWriter() -#include "pycore_object.h" // _PyObject_GC_TRACK() +#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError() #include "pycore_pathconfig.h" // _Py_DumpPathConfig() -#include "pycore_pyerrors.h" // _Py_FatalRefcountError() #include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI @@ -15369,6 +15368,13 @@ onError: return NULL; } +void +_PyUnicode_ExactDealloc(PyObject *op) +{ + assert(PyUnicode_CheckExact(op)); + unicode_dealloc(op); +} + PyDoc_STRVAR(unicode_doc, "str(object='') -> str\n\ str(bytes_or_buffer[, encoding[, errors]]) -> str\n\ diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c index 9cfecc521d9..84ebb680e0b 100644 --- a/Python/bltinmodule.c +++ b/Python/bltinmodule.c @@ -2511,7 +2511,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start) } if (PyFloat_CheckExact(item)) { f_result += PyFloat_AS_DOUBLE(item); - Py_DECREF(item); + _Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc); continue; } if (PyLong_Check(item)) { diff --git a/Python/ceval.c b/Python/ceval.c index f523e52fe01..45754ffbe7c 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1977,8 +1977,8 @@ handle_eval_breaker: STAT_INC(BINARY_OP, hit); PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(prod); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -1998,8 +1998,8 @@ handle_eval_breaker: ((PyFloatObject *)right)->ob_fval; PyObject *prod = PyFloat_FromDouble(dprod); SET_SECOND(prod); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); STACK_SHRINK(1); if (prod == NULL) { goto error; @@ -2017,8 +2017,8 @@ handle_eval_breaker: STAT_INC(BINARY_OP, hit); PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sub); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2037,8 +2037,8 @@ handle_eval_breaker: double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval; PyObject *sub = PyFloat_FromDouble(dsub); SET_SECOND(sub); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); STACK_SHRINK(1); if (sub == NULL) { goto error; @@ -2057,8 +2057,8 @@ handle_eval_breaker: PyObject *res = PyUnicode_Concat(left, right); STACK_SHRINK(1); SET_TOP(res); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (TOP() == NULL) { goto error; } @@ -2090,10 +2090,10 @@ handle_eval_breaker: * that the string is safe to mutate. */ assert(Py_REFCNT(left) >= 2); - Py_DECREF(left); // XXX never need to dealloc + _Py_DECREF_NO_DEALLOC(left); STACK_SHRINK(2); PyUnicode_Append(target_local, right); - Py_DECREF(right); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); if (*target_local == NULL) { goto error; } @@ -2113,8 +2113,8 @@ handle_eval_breaker: ((PyFloatObject *)right)->ob_fval; PyObject *sum = PyFloat_FromDouble(dsum); SET_SECOND(sum); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2132,8 +2132,8 @@ handle_eval_breaker: STAT_INC(BINARY_OP, hit); PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right); SET_SECOND(sum); - Py_DECREF(right); - Py_DECREF(left); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); STACK_SHRINK(1); if (sum == NULL) { goto error; @@ -2192,7 +2192,7 @@ handle_eval_breaker: assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - Py_DECREF(sub); + _Py_DECREF_SPECIALIZED(sub, PyObject_Free); SET_TOP(res); Py_DECREF(list); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -2217,7 +2217,7 @@ handle_eval_breaker: assert(res != NULL); Py_INCREF(res); STACK_SHRINK(1); - Py_DECREF(sub); + _Py_DECREF_SPECIALIZED(sub, PyObject_Free); SET_TOP(res); Py_DECREF(tuple); JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR); @@ -2359,7 +2359,7 @@ handle_eval_breaker: STACK_SHRINK(3); assert(old_value != NULL); Py_DECREF(old_value); - Py_DECREF(sub); + _Py_DECREF_SPECIALIZED(sub, PyObject_Free); Py_DECREF(list); JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR); NOTRACE_DISPATCH(); @@ -3752,8 +3752,8 @@ handle_eval_breaker: JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); NEXTOPARG(); STACK_SHRINK(2); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); assert(opcode == POP_JUMP_FORWARD_IF_FALSE || opcode == POP_JUMP_BACKWARD_IF_FALSE || opcode == POP_JUMP_FORWARD_IF_TRUE || @@ -3795,8 +3795,8 @@ handle_eval_breaker: JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP); NEXTOPARG(); STACK_SHRINK(2); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_SPECIALIZED(left, PyObject_Free); + _Py_DECREF_SPECIALIZED(right, PyObject_Free); assert(opcode == POP_JUMP_FORWARD_IF_FALSE || opcode == POP_JUMP_BACKWARD_IF_FALSE || opcode == POP_JUMP_FORWARD_IF_TRUE || @@ -3841,8 +3841,8 @@ handle_eval_breaker: opcode == POP_JUMP_FORWARD_IF_TRUE || opcode == POP_JUMP_BACKWARD_IF_TRUE); STACK_SHRINK(2); - Py_DECREF(left); - Py_DECREF(right); + _Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc); assert(res == 0 || res == 1); int sign = 1 - res; int jump = (9 << (sign + 1)) & when_to_jump_mask; @@ -4008,11 +4008,11 @@ handle_eval_breaker: PREDICTED(POP_JUMP_BACKWARD_IF_FALSE); PyObject *cond = POP(); if (Py_IsTrue(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsFalse(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4034,10 +4034,10 @@ handle_eval_breaker: PREDICTED(POP_JUMP_FORWARD_IF_FALSE); PyObject *cond = POP(); if (Py_IsTrue(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); } else if (Py_IsFalse(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(oparg); } else { @@ -4057,11 +4057,11 @@ handle_eval_breaker: TARGET(POP_JUMP_BACKWARD_IF_TRUE) { PyObject *cond = POP(); if (Py_IsFalse(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsTrue(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); DISPATCH(); @@ -4082,10 +4082,10 @@ handle_eval_breaker: TARGET(POP_JUMP_FORWARD_IF_TRUE) { PyObject *cond = POP(); if (Py_IsFalse(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); } else if (Py_IsTrue(cond)) { - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); JUMPBY(oparg); } else { @@ -4110,7 +4110,7 @@ handle_eval_breaker: CHECK_EVAL_BREAKER(); DISPATCH(); } - Py_DECREF(value); + _Py_DECREF_NO_DEALLOC(value); DISPATCH(); } @@ -4126,21 +4126,25 @@ handle_eval_breaker: TARGET(POP_JUMP_BACKWARD_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { - Py_DECREF(value); + _Py_DECREF_NO_DEALLOC(value); JUMPBY(-oparg); CHECK_EVAL_BREAKER(); - DISPATCH(); } - Py_DECREF(value); + else { + Py_DECREF(value); + } DISPATCH(); } TARGET(POP_JUMP_FORWARD_IF_NONE) { PyObject *value = POP(); if (Py_IsNone(value)) { + _Py_DECREF_NO_DEALLOC(value); JUMPBY(oparg); } - Py_DECREF(value); + else { + Py_DECREF(value); + } DISPATCH(); } @@ -4149,7 +4153,7 @@ handle_eval_breaker: int err; if (Py_IsTrue(cond)) { STACK_SHRINK(1); - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsFalse(cond)) { @@ -4173,7 +4177,7 @@ handle_eval_breaker: int err; if (Py_IsFalse(cond)) { STACK_SHRINK(1); - Py_DECREF(cond); + _Py_DECREF_NO_DEALLOC(cond); DISPATCH(); } if (Py_IsTrue(cond)) {