GH-124284: Add stats for refcount operations on immortal objects (GH-124288)

Include/cpython/pystats.h

@@ -70,6 +70,10 @@ typedef struct _object_stats {
     uint64_t decrefs;
     uint64_t interpreter_increfs;
     uint64_t interpreter_decrefs;
+    uint64_t immortal_increfs;
+    uint64_t immortal_decrefs;
+    uint64_t interpreter_immortal_increfs;
+    uint64_t interpreter_immortal_decrefs;
     uint64_t allocations;
     uint64_t allocations512;
     uint64_t allocations4k;
@@ -163,7 +167,11 @@ PyAPI_DATA(PyStats*) _Py_stats;
 #ifdef _PY_INTERPRETER
 #  define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0)
 #  define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0)
+#  define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_increfs++; } while (0)
+#  define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_immortal_decrefs++; } while (0)
 #else
 #  define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0)
 #  define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0)
+#  define _Py_INCREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_increfs++; } while (0)
+#  define _Py_DECREF_IMMORTAL_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.immortal_decrefs++; } while (0)
 #endif

Include/internal/pycore_object.h

@@ -214,6 +214,7 @@ static inline void
 _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
 {
     if (_Py_IsImmortal(op)) {
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
     _Py_DECREF_STAT_INC();
@@ -235,6 +236,7 @@ static inline void
 _Py_DECREF_NO_DEALLOC(PyObject *op)
 {
     if (_Py_IsImmortal(op)) {
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
     _Py_DECREF_STAT_INC();
@@ -315,6 +317,7 @@ _Py_INCREF_TYPE(PyTypeObject *type)
 {
     if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
         assert(_Py_IsImmortalLoose(type));
+        _Py_INCREF_IMMORTAL_STAT_INC();
         return;
     }
 
@@ -355,6 +358,7 @@ _Py_DECREF_TYPE(PyTypeObject *type)
 {
     if (!_PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
         assert(_Py_IsImmortalLoose(type));
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
 
@@ -511,6 +515,7 @@ _Py_TryIncrefFast(PyObject *op) {
     local += 1;
     if (local == 0) {
         // immortal
+        _Py_INCREF_IMMORTAL_STAT_INC();
         return 1;
     }
     if (_Py_IsOwnedByCurrentThread(op)) {

Include/pystats.h

@@ -18,6 +18,8 @@ extern "C" {
 #else
 #  define _Py_INCREF_STAT_INC() ((void)0)
 #  define _Py_DECREF_STAT_INC() ((void)0)
+#  define _Py_INCREF_IMMORTAL_STAT_INC() ((void)0)
+#  define _Py_DECREF_IMMORTAL_STAT_INC() ((void)0)
 #endif  // !Py_STATS
 
 #ifdef __cplusplus

Include/refcount.h

@@ -227,6 +227,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
     uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
     uint32_t new_local = local + 1;
     if (new_local == 0) {
+        _Py_INCREF_IMMORTAL_STAT_INC();
         // local is equal to _Py_IMMORTAL_REFCNT: do nothing
         return;
     }
@@ -241,6 +242,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
     PY_UINT32_T cur_refcnt = op->ob_refcnt_split[PY_BIG_ENDIAN];
     PY_UINT32_T new_refcnt = cur_refcnt + 1;
     if (new_refcnt == 0) {
+        _Py_INCREF_IMMORTAL_STAT_INC();
         // cur_refcnt is equal to _Py_IMMORTAL_REFCNT: the object is immortal,
         // do nothing
         return;
@@ -249,6 +251,7 @@ static inline Py_ALWAYS_INLINE void Py_INCREF(PyObject *op)
 #else
     // Explicitly check immortality against the immortal value
     if (_Py_IsImmortal(op)) {
+        _Py_INCREF_IMMORTAL_STAT_INC();
         return;
     }
     op->ob_refcnt++;
@@ -295,6 +298,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
 {
     uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
     if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
     _Py_DECREF_STAT_INC();
@@ -320,6 +324,7 @@ static inline void Py_DECREF(PyObject *op)
 {
     uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
     if (local == _Py_IMMORTAL_REFCNT_LOCAL) {
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
     _Py_DECREF_STAT_INC();
@@ -343,6 +348,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
         _Py_NegativeRefcount(filename, lineno, op);
     }
     if (_Py_IsImmortal(op)) {
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
     _Py_DECREF_STAT_INC();
@@ -359,6 +365,7 @@ static inline Py_ALWAYS_INLINE void Py_DECREF(PyObject *op)
     // Non-limited C API and limited C API for Python 3.9 and older access
     // directly PyObject.ob_refcnt.
     if (_Py_IsImmortal(op)) {
+        _Py_DECREF_IMMORTAL_STAT_INC();
         return;
     }
     _Py_DECREF_STAT_INC();

Python/ceval.c

@@ -79,6 +79,7 @@
     do { \
         PyObject *op = _PyObject_CAST(arg); \
         if (_Py_IsImmortal(op)) { \
+            _Py_DECREF_IMMORTAL_STAT_INC(); \
             break; \
         } \
         _Py_DECREF_STAT_INC(); \
@@ -93,6 +94,7 @@
     do { \
         PyObject *op = _PyObject_CAST(arg); \
         if (_Py_IsImmortal(op)) { \
+            _Py_DECREF_IMMORTAL_STAT_INC(); \
             break; \
         } \
         _Py_DECREF_STAT_INC(); \
@@ -110,6 +112,7 @@
         PyObject *op = _PyObject_CAST(arg); \
         uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local); \
         if (local == _Py_IMMORTAL_REFCNT_LOCAL) { \
+            _Py_DECREF_IMMORTAL_STAT_INC(); \
             break; \
         } \
         _Py_DECREF_STAT_INC(); \

Python/specialize.c

@@ -205,10 +205,14 @@ print_object_stats(FILE *out, ObjectStats *stats)
     fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
     fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
     fprintf(out, "Object inline values: %" PRIu64 "\n", stats->inline_values);
-    fprintf(out, "Object interpreter increfs: %" PRIu64 "\n", stats->interpreter_increfs);
+    fprintf(out, "Object interpreter mortal increfs: %" PRIu64 "\n", stats->interpreter_increfs);
-    fprintf(out, "Object interpreter decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
+    fprintf(out, "Object interpreter mortal decrefs: %" PRIu64 "\n", stats->interpreter_decrefs);
-    fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
+    fprintf(out, "Object mortal increfs: %" PRIu64 "\n", stats->increfs);
-    fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
+    fprintf(out, "Object mortal decrefs: %" PRIu64 "\n", stats->decrefs);
+    fprintf(out, "Object interpreter immortal increfs: %" PRIu64 "\n", stats->interpreter_immortal_increfs);
+    fprintf(out, "Object interpreter immortal decrefs: %" PRIu64 "\n", stats->interpreter_immortal_decrefs);
+    fprintf(out, "Object immortal increfs: %" PRIu64 "\n", stats->immortal_increfs);
+    fprintf(out, "Object immortal decrefs: %" PRIu64 "\n", stats->immortal_decrefs);
     fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
     fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
     fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);

Tools/scripts/summarize_stats.py

@@ -398,12 +398,18 @@ class Stats:
         total_allocations = self._data.get("Object allocations", 0) + self._data.get(
             "Object allocations from freelist", 0
         )
-        total_increfs = self._data.get(
+        total_increfs = (
-            "Object interpreter increfs", 0
+            self._data.get("Object interpreter mortal increfs", 0) +
-        ) + self._data.get("Object increfs", 0)
+            self._data.get("Object mortal increfs", 0) +
-        total_decrefs = self._data.get(
+            self._data.get("Object interpreter immortal increfs", 0) +
-            "Object interpreter decrefs", 0
+            self._data.get("Object immortal increfs", 0)
-        ) + self._data.get("Object decrefs", 0)
+        )
+        total_decrefs = (
+            self._data.get("Object interpreter mortal decrefs", 0) +
+            self._data.get("Object mortal decrefs", 0) +
+            self._data.get("Object interpreter immortal decrefs", 0) +
+            self._data.get("Object immortal decrefs", 0)
+        )
 
         result = {}
         for key, value in self._data.items():