Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-90667: Add specializations of Py_DECREF when types are known (GH-30872)

This commit is contained in:
Dennis Sweeney 2022-04-19 14:02:19 -04:00 committed by GitHub
parent ab0d35d70d
commit da6c78584b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 147 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Include/internal/pycore_floatobject.h
View File

@ -38,6 +38,8 @@ struct _Py_float_state {
#endif
};
void _PyFloat_ExactDealloc(PyObject *op);
PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out);

37
Include/internal/pycore_object.h
View File

@ -14,7 +14,6 @@ extern "C" {
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_runtime.h" // _PyRuntime
#define _PyObject_IMMORTAL_INIT(type) \
{ \
.ob_refcnt = 999999999, \
@ -26,6 +25,42 @@ extern "C" {
.ob_size = size, \
}
PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
const char *func,
const char *message);
#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message)
static inline void
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
{
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif
if (--op->ob_refcnt != 0) {
assert(op->ob_refcnt > 0);
}
else {
#ifdef Py_TRACE_REFS
_Py_ForgetReference(op);
#endif
destruct(op);
}
}
static inline void
_Py_DECREF_NO_DEALLOC(PyObject *op)
{
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif
op->ob_refcnt--;
#ifdef Py_DEBUG
if (op->ob_refcnt <= 0) {
_Py_FatalRefcountError("Expected a positive remaining refcount");
}
#endif
}
PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type);
PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);

7
Include/internal/pycore_pyerrors.h
View File

@ -100,13 +100,6 @@ extern PyObject* _Py_Offer_Suggestions(PyObject* exception);
PyAPI_FUNC(Py_ssize_t) _Py_UTF8_Edit_Cost(PyObject *str_a, PyObject *str_b,
Py_ssize_t max_cost);
PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
const char *func,
const char *message);
#define _Py_FatalRefcountError(message) _Py_FatalRefcountErrorFunc(__func__, message)
#ifdef __cplusplus
}
#endif

1
Include/internal/pycore_unicodeobject.h
View File

@ -10,6 +10,7 @@ extern "C" {
#include "pycore_fileutils.h" // _Py_error_handler
void _PyUnicode_ExactDealloc(PyObject *op);
/* runtime lifecycle */

1
Misc/NEWS.d/next/Core and Builtins/2022-01-25-05-39-38.bpo-46509.ljrqrc.rst Normal file
View File

@ -0,0 +1 @@
Add type-specialized versions of the ``Py_DECREF()``, and use them for ``float``, ``int``, ``str``, ``bool``, and ``None`` to avoid pointer-chasing at runtime where types are known at C compile time.

2
Objects/boolobject.c
View File

@ -1,8 +1,8 @@
/* Boolean type, a subtype of int */
#include "Python.h"
#include "pycore_object.h" // _Py_FatalRefcountError()
#include "pycore_runtime.h" // _Py_ID()
#include "pycore_pyerrors.h" // _Py_FatalRefcountError()
/* We define bool_repr to return "False" or "True" */

21
Objects/floatobject.c
View File

@ -238,11 +238,12 @@ PyFloat_FromString(PyObject *v)
return result;
}
static void
float_dealloc(PyFloatObject *op)
void
_PyFloat_ExactDealloc(PyObject *obj)
{
assert(PyFloat_CheckExact(obj));
PyFloatObject *op = (PyFloatObject *)obj;
#if PyFloat_MAXFREELIST > 0
if (PyFloat_CheckExact(op)) {
struct _Py_float_state *state = get_float_state();
#ifdef Py_DEBUG
// float_dealloc() must not be called after _PyFloat_Fini()
@ -255,11 +256,23 @@ float_dealloc(PyFloatObject *op)
state->numfree++;
Py_SET_TYPE(op, (PyTypeObject *)state->free_list);
state->free_list = op;
#else
PyObject_Free(op);
#endif
}
static void
float_dealloc(PyObject *op)
{
assert(PyFloat_Check(op));
#if PyFloat_MAXFREELIST > 0
if (PyFloat_CheckExact(op)) {
_PyFloat_ExactDealloc(op);
}
else
#endif
{
Py_TYPE(op)->tp_free((PyObject *)op);
Py_TYPE(op)->tp_free(op);
}
}

38
Objects/longobject.c
View File

@ -36,7 +36,15 @@ medium_value(PyLongObject *x)
#define IS_SMALL_INT(ival) (-_PY_NSMALLNEGINTS <= (ival) && (ival) < _PY_NSMALLPOSINTS)
#define IS_SMALL_UINT(ival) ((ival) < _PY_NSMALLPOSINTS)
static inline int is_medium_int(stwodigits x)
static inline void
_Py_DECREF_INT(PyLongObject *op)
{
assert(PyLong_CheckExact(op));
_Py_DECREF_SPECIALIZED((PyObject *)op, PyObject_Free);
}
static inline int
is_medium_int(stwodigits x)
{
/* Take care that we are comparing unsigned values. */
twodigits x_plus_mask = ((twodigits)x) + PyLong_MASK;
@ -58,7 +66,7 @@ maybe_small_long(PyLongObject *v)
if (v && IS_MEDIUM_VALUE(v)) {
stwodigits ival = medium_value(v);
if (IS_SMALL_INT(ival)) {
Py_DECREF(v);
_Py_DECREF_INT(v);
return (PyLongObject *)get_small_int((sdigit)ival);
}
}
@ -1856,7 +1864,7 @@ long_to_decimal_string_internal(PyObject *aa,
#undef WRITE_DIGITS
#undef WRITE_UNICODE_DIGITS
Py_DECREF(scratch);
_Py_DECREF_INT(scratch);
if (writer) {
writer->pos += strlen;
}
@ -3561,15 +3569,15 @@ k_mul(PyLongObject *a, PyLongObject *b)
*/
i = Py_SIZE(ret) - shift; /* # digits after shift */
(void)v_isub(ret->ob_digit + shift, i, t2->ob_digit, Py_SIZE(t2));
Py_DECREF(t2);
_Py_DECREF_INT(t2);
(void)v_isub(ret->ob_digit + shift, i, t1->ob_digit, Py_SIZE(t1));
Py_DECREF(t1);
_Py_DECREF_INT(t1);
/* 6. t3 <- (ah+al)(bh+bl), and add into result. */
if ((t1 = x_add(ah, al)) == NULL) goto fail;
Py_DECREF(ah);
Py_DECREF(al);
_Py_DECREF_INT(ah);
_Py_DECREF_INT(al);
ah = al = NULL;
if (a == b) {
@ -3580,13 +3588,13 @@ k_mul(PyLongObject *a, PyLongObject *b)
Py_DECREF(t1);
goto fail;
}
Py_DECREF(bh);
Py_DECREF(bl);
_Py_DECREF_INT(bh);
_Py_DECREF_INT(bl);
bh = bl = NULL;
t3 = k_mul(t1, t2);
Py_DECREF(t1);
Py_DECREF(t2);
_Py_DECREF_INT(t1);
_Py_DECREF_INT(t2);
if (t3 == NULL) goto fail;
assert(Py_SIZE(t3) >= 0);
@ -3594,7 +3602,7 @@ k_mul(PyLongObject *a, PyLongObject *b)
* See the (*) comment after this function.
*/
(void)v_iadd(ret->ob_digit + shift, i, t3->ob_digit, Py_SIZE(t3));
Py_DECREF(t3);
_Py_DECREF_INT(t3);
return long_normalize(ret);
@ -3699,13 +3707,13 @@ k_lopsided_mul(PyLongObject *a, PyLongObject *b)
/* Add into result. */
(void)v_iadd(ret->ob_digit + nbdone, Py_SIZE(ret) - nbdone,
product->ob_digit, Py_SIZE(product));
Py_DECREF(product);
_Py_DECREF_INT(product);
bsize -= nbtouse;
nbdone += nbtouse;
}
Py_DECREF(bslice);
_Py_DECREF_INT(bslice);
return long_normalize(ret);
fail:
@ -5993,7 +6001,7 @@ PyTypeObject PyLong_Type = {
0, /* tp_init */
0, /* tp_alloc */
long_new, /* tp_new */
PyObject_Del, /* tp_free */
PyObject_Free, /* tp_free */
};
static PyTypeObject Int_InfoType;

2
Objects/object.c
View File

@ -9,7 +9,7 @@
#include "pycore_floatobject.h" // _PyFloat_DebugMallocStats()
#include "pycore_initconfig.h" // _PyStatus_EXCEPTION()
#include "pycore_namespace.h" // _PyNamespace_Type
#include "pycore_object.h" // _PyType_CheckConsistency()
#include "pycore_object.h" // _PyType_CheckConsistency(), _Py_FatalRefcountError()
#include "pycore_pyerrors.h" // _PyErr_Occurred()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_pystate.h" // _PyThreadState_GET()

3
Objects/tupleobject.c
View File

@ -5,8 +5,7 @@
#include "pycore_abstract.h" // _PyIndex_Check()
#include "pycore_gc.h" // _PyObject_GC_IS_TRACKED()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_pyerrors.h" // _Py_FatalRefcountError()
#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError()
/*[clinic input]
class tuple "PyTupleObject *" "&PyTuple_Type"

10
Objects/unicodeobject.c
View File

@ -48,9 +48,8 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_interp.h" // PyInterpreterState.fs_codec
#include "pycore_long.h" // _PyLong_FormatWriter()
#include "pycore_object.h" // _PyObject_GC_TRACK()
#include "pycore_object.h" // _PyObject_GC_TRACK(), _Py_FatalRefcountError()
#include "pycore_pathconfig.h" // _Py_DumpPathConfig()
#include "pycore_pyerrors.h" // _Py_FatalRefcountError()
#include "pycore_pylifecycle.h" // _Py_SetFileSystemEncoding()
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI
@ -15369,6 +15368,13 @@ onError:
return NULL;
}
void
_PyUnicode_ExactDealloc(PyObject *op)
{
assert(PyUnicode_CheckExact(op));
unicode_dealloc(op);
}
PyDoc_STRVAR(unicode_doc,
"str(object='') -> str\n\
str(bytes_or_buffer[, encoding[, errors]]) -> str\n\

2
Python/bltinmodule.c
View File

@ -2511,7 +2511,7 @@ builtin_sum_impl(PyObject *module, PyObject *iterable, PyObject *start)
}
if (PyFloat_CheckExact(item)) {
f_result += PyFloat_AS_DOUBLE(item);
Py_DECREF(item);
_Py_DECREF_SPECIALIZED(item, _PyFloat_ExactDealloc);
continue;
}
if (PyLong_Check(item)) {

80
Python/ceval.c
View File

@ -1977,8 +1977,8 @@ handle_eval_breaker:
STAT_INC(BINARY_OP, hit);
PyObject *prod = _PyLong_Multiply((PyLongObject *)left, (PyLongObject *)right);
SET_SECOND(prod);
Py_DECREF(right);
Py_DECREF(left);
_Py_DECREF_SPECIALIZED(right, PyObject_Free);
_Py_DECREF_SPECIALIZED(left, PyObject_Free);
STACK_SHRINK(1);
if (prod == NULL) {
goto error;
@ -1998,8 +1998,8 @@ handle_eval_breaker:
((PyFloatObject *)right)->ob_fval;
PyObject *prod = PyFloat_FromDouble(dprod);
SET_SECOND(prod);
Py_DECREF(right);
Py_DECREF(left);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
STACK_SHRINK(1);
if (prod == NULL) {
goto error;
@ -2017,8 +2017,8 @@ handle_eval_breaker:
STAT_INC(BINARY_OP, hit);
PyObject *sub = _PyLong_Subtract((PyLongObject *)left, (PyLongObject *)right);
SET_SECOND(sub);
Py_DECREF(right);
Py_DECREF(left);
_Py_DECREF_SPECIALIZED(right, PyObject_Free);
_Py_DECREF_SPECIALIZED(left, PyObject_Free);
STACK_SHRINK(1);
if (sub == NULL) {
goto error;
@ -2037,8 +2037,8 @@ handle_eval_breaker:
double dsub = ((PyFloatObject *)left)->ob_fval - ((PyFloatObject *)right)->ob_fval;
PyObject *sub = PyFloat_FromDouble(dsub);
SET_SECOND(sub);
Py_DECREF(right);
Py_DECREF(left);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
STACK_SHRINK(1);
if (sub == NULL) {
goto error;
@ -2057,8 +2057,8 @@ handle_eval_breaker:
PyObject *res = PyUnicode_Concat(left, right);
STACK_SHRINK(1);
SET_TOP(res);
Py_DECREF(left);
Py_DECREF(right);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (TOP() == NULL) {
goto error;
}
@ -2090,10 +2090,10 @@ handle_eval_breaker:
* that the string is safe to mutate.
*/
assert(Py_REFCNT(left) >= 2);
Py_DECREF(left); // XXX never need to dealloc
_Py_DECREF_NO_DEALLOC(left);
STACK_SHRINK(2);
PyUnicode_Append(target_local, right);
Py_DECREF(right);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
if (*target_local == NULL) {
goto error;
}
@ -2113,8 +2113,8 @@ handle_eval_breaker:
((PyFloatObject *)right)->ob_fval;
PyObject *sum = PyFloat_FromDouble(dsum);
SET_SECOND(sum);
Py_DECREF(right);
Py_DECREF(left);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
STACK_SHRINK(1);
if (sum == NULL) {
goto error;
@ -2132,8 +2132,8 @@ handle_eval_breaker:
STAT_INC(BINARY_OP, hit);
PyObject *sum = _PyLong_Add((PyLongObject *)left, (PyLongObject *)right);
SET_SECOND(sum);
Py_DECREF(right);
Py_DECREF(left);
_Py_DECREF_SPECIALIZED(right, PyObject_Free);
_Py_DECREF_SPECIALIZED(left, PyObject_Free);
STACK_SHRINK(1);
if (sum == NULL) {
goto error;
@ -2192,7 +2192,7 @@ handle_eval_breaker:
assert(res != NULL);
Py_INCREF(res);
STACK_SHRINK(1);
Py_DECREF(sub);
_Py_DECREF_SPECIALIZED(sub, PyObject_Free);
SET_TOP(res);
Py_DECREF(list);
JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
@ -2217,7 +2217,7 @@ handle_eval_breaker:
assert(res != NULL);
Py_INCREF(res);
STACK_SHRINK(1);
Py_DECREF(sub);
_Py_DECREF_SPECIALIZED(sub, PyObject_Free);
SET_TOP(res);
Py_DECREF(tuple);
JUMPBY(INLINE_CACHE_ENTRIES_BINARY_SUBSCR);
@ -2359,7 +2359,7 @@ handle_eval_breaker:
STACK_SHRINK(3);
assert(old_value != NULL);
Py_DECREF(old_value);
Py_DECREF(sub);
_Py_DECREF_SPECIALIZED(sub, PyObject_Free);
Py_DECREF(list);
JUMPBY(INLINE_CACHE_ENTRIES_STORE_SUBSCR);
NOTRACE_DISPATCH();
@ -3752,8 +3752,8 @@ handle_eval_breaker:
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
Py_DECREF(left);
Py_DECREF(right);
_Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc);
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
@ -3795,8 +3795,8 @@ handle_eval_breaker:
JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);
NEXTOPARG();
STACK_SHRINK(2);
Py_DECREF(left);
Py_DECREF(right);
_Py_DECREF_SPECIALIZED(left, PyObject_Free);
_Py_DECREF_SPECIALIZED(right, PyObject_Free);
assert(opcode == POP_JUMP_FORWARD_IF_FALSE ||
opcode == POP_JUMP_BACKWARD_IF_FALSE ||
opcode == POP_JUMP_FORWARD_IF_TRUE ||
@ -3841,8 +3841,8 @@ handle_eval_breaker:
opcode == POP_JUMP_FORWARD_IF_TRUE ||
opcode == POP_JUMP_BACKWARD_IF_TRUE);
STACK_SHRINK(2);
Py_DECREF(left);
Py_DECREF(right);
_Py_DECREF_SPECIALIZED(left, _PyUnicode_ExactDealloc);
_Py_DECREF_SPECIALIZED(right, _PyUnicode_ExactDealloc);
assert(res == 0 || res == 1);
int sign = 1 - res;
int jump = (9 << (sign + 1)) & when_to_jump_mask;
@ -4008,11 +4008,11 @@ handle_eval_breaker:
PREDICTED(POP_JUMP_BACKWARD_IF_FALSE);
PyObject *cond = POP();
if (Py_IsTrue(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
DISPATCH();
}
if (Py_IsFalse(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
@ -4034,10 +4034,10 @@ handle_eval_breaker:
PREDICTED(POP_JUMP_FORWARD_IF_FALSE);
PyObject *cond = POP();
if (Py_IsTrue(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
}
else if (Py_IsFalse(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
JUMPBY(oparg);
}
else {
@ -4057,11 +4057,11 @@ handle_eval_breaker:
TARGET(POP_JUMP_BACKWARD_IF_TRUE) {
PyObject *cond = POP();
if (Py_IsFalse(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
DISPATCH();
}
if (Py_IsTrue(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
@ -4082,10 +4082,10 @@ handle_eval_breaker:
TARGET(POP_JUMP_FORWARD_IF_TRUE) {
PyObject *cond = POP();
if (Py_IsFalse(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
}
else if (Py_IsTrue(cond)) {
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
JUMPBY(oparg);
}
else {
@ -4110,7 +4110,7 @@ handle_eval_breaker:
CHECK_EVAL_BREAKER();
DISPATCH();
}
Py_DECREF(value);
_Py_DECREF_NO_DEALLOC(value);
DISPATCH();
}
@ -4126,21 +4126,25 @@ handle_eval_breaker:
TARGET(POP_JUMP_BACKWARD_IF_NONE) {
PyObject *value = POP();
if (Py_IsNone(value)) {
Py_DECREF(value);
_Py_DECREF_NO_DEALLOC(value);
JUMPBY(-oparg);
CHECK_EVAL_BREAKER();
DISPATCH();
}
else {
Py_DECREF(value);
}
DISPATCH();
}
TARGET(POP_JUMP_FORWARD_IF_NONE) {
PyObject *value = POP();
if (Py_IsNone(value)) {
_Py_DECREF_NO_DEALLOC(value);
JUMPBY(oparg);
}
else {
Py_DECREF(value);
}
DISPATCH();
}
@ -4149,7 +4153,7 @@ handle_eval_breaker:
int err;
if (Py_IsTrue(cond)) {
STACK_SHRINK(1);
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
DISPATCH();
}
if (Py_IsFalse(cond)) {
@ -4173,7 +4177,7 @@ handle_eval_breaker:
int err;
if (Py_IsFalse(cond)) {
STACK_SHRINK(1);
Py_DECREF(cond);
_Py_DECREF_NO_DEALLOC(cond);
DISPATCH();
}
if (Py_IsTrue(cond)) {