Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

Improve object stats (#92845) 

* Add incref/decref stats

* Show ratios for allocation in summary
This commit is contained in:
Mark Shannon 2022-05-16 14:35:11 +01:00 committed by GitHub
parent f6fd8aac13
commit fa2b8b75eb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 106 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
Include/internal/pycore_code.h
View File

@ -265,53 +265,6 @@ extern int _PyStaticCode_InternStrings(PyCodeObject *co);
#ifdef Py_STATS
#define SPECIALIZATION_FAILURE_KINDS 30
typedef struct _specialization_stats {
uint64_t success;
uint64_t failure;
uint64_t hit;
uint64_t deferred;
uint64_t miss;
uint64_t deopt;
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
} SpecializationStats;
typedef struct _opcode_stats {
SpecializationStats specialization;
uint64_t execution_count;
uint64_t pair_count[256];
} OpcodeStats;
typedef struct _call_stats {
uint64_t inlined_py_calls;
uint64_t pyeval_calls;
uint64_t frames_pushed;
uint64_t frame_objects_created;
} CallStats;
typedef struct _object_stats {
uint64_t allocations;
uint64_t allocations512;
uint64_t allocations4k;
uint64_t allocations_big;
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
} ObjectStats;
typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
} PyStats;
extern PyStats _py_stats;
#define STAT_INC(opname, name) _py_stats.opcode_stats[opname].specialization.name++
#define STAT_DEC(opname, name) _py_stats.opcode_stats[opname].specialization.name--
@ -321,8 +274,6 @@ extern PyStats _py_stats;
#define OBJECT_STAT_INC_COND(name, cond) \
do { if (cond) _py_stats.object_stats.name++; } while (0)
extern void _Py_PrintSpecializationStats(int to_file);
// Used by the _opcode extension which is built as a shared library
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);

2
Include/internal/pycore_object.h
View File

@ -34,6 +34,7 @@ PyAPI_FUNC(void) _Py_NO_RETURN _Py_FatalRefcountErrorFunc(
static inline void
_Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
{
_Py_DECREF_STAT_INC();
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif
@ -51,6 +52,7 @@ _Py_DECREF_SPECIALIZED(PyObject *op, const destructor destruct)
static inline void
_Py_DECREF_NO_DEALLOC(PyObject *op)
{
_Py_DECREF_STAT_INC();
#ifdef Py_REF_DEBUG
_Py_RefTotal--;
#endif

6
Include/object.h
View File

@ -51,6 +51,8 @@ A standard interface exists for objects that contain an array of items
whose size is determined when the object is allocated.
*/
#include "pystats.h"
/* Py_DEBUG implies Py_REF_DEBUG. */
#if defined(Py_DEBUG) && !defined(Py_REF_DEBUG)
# define Py_REF_DEBUG
@ -490,6 +492,7 @@ PyAPI_FUNC(void) _Py_DecRef(PyObject *);
static inline void Py_INCREF(PyObject *op)
{
_Py_INCREF_STAT_INC();
#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
// Stable ABI for Python 3.10 built in debug mode.
_Py_IncRef(op);
@ -506,7 +509,6 @@ static inline void Py_INCREF(PyObject *op)
# define Py_INCREF(op) Py_INCREF(_PyObject_CAST(op))
#endif
#if defined(Py_REF_DEBUG) && defined(Py_LIMITED_API) && Py_LIMITED_API+0 >= 0x030A0000
// Stable ABI for limited C API version 3.10 of Python debug build
static inline void Py_DECREF(PyObject *op) {
@ -517,6 +519,7 @@ static inline void Py_DECREF(PyObject *op) {
#elif defined(Py_REF_DEBUG)
static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
{
_Py_DECREF_STAT_INC();
_Py_RefTotal--;
if (--op->ob_refcnt != 0) {
if (op->ob_refcnt < 0) {
@ -532,6 +535,7 @@ static inline void Py_DECREF(const char *filename, int lineno, PyObject *op)
#else
static inline void Py_DECREF(PyObject *op)
{
_Py_DECREF_STAT_INC();
// Non-limited C API and limited C API for Python 3.9 and older access
// directly PyObject.ob_refcnt.
if (--op->ob_refcnt == 0) {

77
Include/pystats.h Normal file
View File

@ -0,0 +1,77 @@
#ifndef Py_PYSTATS_H
#define Py_PYSTATS_H
#ifdef __cplusplus
extern "C" {
#endif
#ifdef Py_STATS
#define SPECIALIZATION_FAILURE_KINDS 32
typedef struct _specialization_stats {
uint64_t success;
uint64_t failure;
uint64_t hit;
uint64_t deferred;
uint64_t miss;
uint64_t deopt;
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
} SpecializationStats;
typedef struct _opcode_stats {
SpecializationStats specialization;
uint64_t execution_count;
uint64_t pair_count[256];
} OpcodeStats;
typedef struct _call_stats {
uint64_t inlined_py_calls;
uint64_t pyeval_calls;
uint64_t frames_pushed;
uint64_t frame_objects_created;
} CallStats;
typedef struct _object_stats {
uint64_t increfs;
uint64_t decrefs;
uint64_t allocations;
uint64_t allocations512;
uint64_t allocations4k;
uint64_t allocations_big;
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
} ObjectStats;
typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
} PyStats;
PyAPI_DATA(PyStats) _py_stats;
extern void _Py_PrintSpecializationStats(int to_file);
#define _Py_INCREF_STAT_INC() _py_stats.object_stats.increfs++
#define _Py_DECREF_STAT_INC() _py_stats.object_stats.decrefs++
#else
#define _Py_INCREF_STAT_INC() ((void)0)
#define _Py_DECREF_STAT_INC() ((void)0)
#endif // !Py_STATS
#ifdef __cplusplus
}
#endif
#endif /* !Py_PYSTATs_H */

1
Makefile.pre.in
View File

@ -1509,6 +1509,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/pymem.h \
$(srcdir)/Include/pyport.h \
$(srcdir)/Include/pystate.h \
$(srcdir)/Include/pystats.h \
$(srcdir)/Include/pystrcmp.h \
$(srcdir)/Include/pystrtod.h \
$(srcdir)/Include/pythonrun.h \

1
PCbuild/pythoncore.vcxproj
View File

@ -280,6 +280,7 @@
<ClInclude Include="..\Include\pymem.h" />
<ClInclude Include="..\Include\pyport.h" />
<ClInclude Include="..\Include\pystate.h" />
<ClInclude Include="..\Include\pystats.h" />
<ClInclude Include="..\Include\pystrcmp.h" />
<ClInclude Include="..\Include\pystrtod.h" />
<ClInclude Include="..\Include\pythonrun.h" />

3
PCbuild/pythoncore.vcxproj.filters
View File

@ -174,6 +174,9 @@
<ClInclude Include="..\Include\pystate.h">
<Filter>Include</Filter>
</ClInclude>
<ClInclude Include="..\Include\pystats.h">
<Filter>Include</Filter>
</ClInclude>
<ClInclude Include="..\Include\pystrcmp.h">
<Filter>Include</Filter>
</ClInclude>

2
Python/ceval.c
View File

@ -55,6 +55,7 @@
#undef Py_DECREF
#define Py_DECREF(arg) \
do { \
_Py_DECREF_STAT_INC(); \
PyObject *op = _PyObject_CAST(arg); \
if (--op->ob_refcnt == 0) { \
destructor dealloc = Py_TYPE(op)->tp_dealloc; \
@ -78,6 +79,7 @@
#undef _Py_DECREF_SPECIALIZED
#define _Py_DECREF_SPECIALIZED(arg, dealloc) \
do { \
_Py_DECREF_STAT_INC(); \
PyObject *op = _PyObject_CAST(arg); \
if (--op->ob_refcnt == 0) { \
destructor d = (destructor)(dealloc); \

2
Python/specialize.c
View File

@ -191,6 +191,8 @@ print_object_stats(FILE *out, ObjectStats *stats)
fprintf(out, "Object allocations over 4 kbytes: %" PRIu64 "\n", stats->allocations_big);
fprintf(out, "Object frees: %" PRIu64 "\n", stats->frees);
fprintf(out, "Object new values: %" PRIu64 "\n", stats->new_values);
fprintf(out, "Object increfs: %" PRIu64 "\n", stats->increfs);
fprintf(out, "Object decrefs: %" PRIu64 "\n", stats->decrefs);
fprintf(out, "Object materialize dict (on request): %" PRIu64 "\n", stats->dict_materialized_on_request);
fprintf(out, "Object materialize dict (new key): %" PRIu64 "\n", stats->dict_materialized_new_key);
fprintf(out, "Object materialize dict (too big): %" PRIu64 "\n", stats->dict_materialized_too_big);

18
Tools/scripts/summarize_stats.py
View File

@ -8,6 +8,7 @@ import opcode
from datetime import date
import itertools
import argparse
import sys
if os.name == "nt":
DEFAULT_DIR = "c:\\temp\\py_stats\\"
@ -88,7 +89,11 @@ def gather_stats():
for filename in os.listdir(DEFAULT_DIR):
with open(os.path.join(DEFAULT_DIR, filename)) as fd:
for line in fd:
key, value = line.split(":")
try:
key, value = line.split(":")
except ValueError:
print (f"Unparsable line: '{line.strip()}' in {filename}", file=sys.stderr)
continue
key = key.strip()
value = int(value)
stats[key] += value
@ -265,17 +270,20 @@ def emit_call_stats(stats):
def emit_object_stats(stats):
with Section("Object stats", summary="allocations, frees and dict materializatons"):
total = stats.get("Object new values")
total_materializations = stats.get("Object new values")
total_allocations = stats.get("Object allocations")
rows = []
for key, value in stats.items():
if key.startswith("Object"):
if "materialize" in key:
materialize = f"{100*value/total:0.1f}%"
ratio = f"{100*value/total_materializations:0.1f}%"
elif "allocations" in key:
ratio = f"{100*value/total_allocations:0.1f}%"
else:
materialize = ""
ratio = ""
label = key[6:].strip()
label = label[0].upper() + label[1:]
rows.append((label, value, materialize))
rows.append((label, value, ratio))
emit_table(("", "Count:", "Ratio:"), rows)
def get_total(opcode_stats):