Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-108753: Enhance pystats (#108754) 

Statistics gathering is now off by default. Use the "-X pystats"
command line option or set the new PYTHONSTATS environment variable
to 1 to turn statistics gathering on at Python startup.

Statistics are no longer dumped at exit if statistics gathering was
off or statistics have been cleared.

Changes:

* Add PYTHONSTATS environment variable.
* sys._stats_dump() now returns False if statistics are not dumped
  because they are all equal to zero.
* Add PyConfig._pystats member.
* Add tests on sys functions and on setting PyConfig._pystats to 1.
* Add Include/cpython/pystats.h and Include/internal/pycore_pystats.h
  header files.
* Rename '_py_stats' variable to '_Py_stats'.
* Exclude Include/cpython/pystats.h from the Py_LIMITED_API.
* Move pystats.h include from object.h to Python.h.
* Add _Py_StatsOn() and _Py_StatsOff() functions. Remove
  '_py_stats_struct' variable from the API: make it static in
  specialize.c.
* Document API in Include/pystats.h and Include/cpython/pystats.h.
* Complete pystats documentation in Doc/using/configure.rst.
* Don't write "all zeros" stats: if _stats_off() and _stats_clear()
  or _stats_dump() were called.
* _PyEval_Fini() now always call _Py_PrintSpecializationStats() which
  does nothing if stats are all zeros.

Co-authored-by: Michael Droettboom <mdboom@gmail.com>
This commit is contained in:
Victor Stinner 2023-09-06 17:54:59 +02:00 committed by GitHub
parent 8ff1142578
commit a0773b89df
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 403 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
Doc/using/configure.rst
View File

@ -192,14 +192,69 @@ General Options
.. cmdoption:: --enable-pystats
Turn on internal statistics gathering.
Turn on internal Python performance statistics gathering.
By default, statistics gathering is off. Use ``python3 -X pystats`` command
or set ``PYTHONSTATS=1`` environment variable to turn on statistics
gathering at Python startup.
At Python exit, dump statistics if statistics gathering was on and not
cleared.
Effects:
* Add :option:`-X pystats <-X>` command line option.
* Add :envvar:`!PYTHONSTATS` environment variable.
* Define the ``Py_STATS`` macro.
* Add functions to the :mod:`sys` module:
* :func:`!sys._stats_on`: Turns on statistics gathering.
* :func:`!sys._stats_off`: Turns off statistics gathering.
* :func:`!sys._stats_clear`: Clears the statistics.
* :func:`!sys._stats_dump`: Dump statistics to file, and clears the statistics.
The statistics will be dumped to a arbitrary (probably unique) file in
``/tmp/py_stats/``, or ``C:\temp\py_stats\`` on Windows. If that directory
does not exist, results will be printed on stdout.
``/tmp/py_stats/`` (Unix) or ``C:\temp\py_stats\`` (Windows). If that
directory does not exist, results will be printed on stderr.
Use ``Tools/scripts/summarize_stats.py`` to read the stats.
Statistics:
* Opcode:
* Specialization: success, failure, hit, deferred, miss, deopt, failures;
* Execution count;
* Pair count.
* Call:
* Inlined Python calls;
* PyEval calls;
* Frames pushed;
* Frame object created;
* Eval calls: vector, generator, legacy, function VECTORCALL, build class,
slot, function "ex", API, method.
* Object:
* incref and decref;
* interpreter incref and decref;
* allocations: all, 512 bytes, 4 kiB, big;
* free;
* to/from free lists;
* dictionary materialized/dematerialized;
* type cache;
* optimization attemps;
* optimization traces created/executed;
* uops executed.
* Garbage collector:
* Garbage collections;
* Objects visited;
* Objects collected.
.. versionadded:: 3.11
.. cmdoption:: --disable-gil

5
Include/cpython/initconfig.h
View File

@ -215,6 +215,11 @@ typedef struct PyConfig {
// If non-zero, we believe we're running from a source tree.
int _is_python_build;
#ifdef Py_STATS
// If non-zero, turns on statistics gathering.
int _pystats;
#endif
} PyConfig;
PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

120
Include/cpython/pystats.h Normal file
View File

@ -0,0 +1,120 @@
// Statistics on Python performance.
//
// API:
//
// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
// and Py_DECREF().
// - _Py_stats variable
//
// Functions of the sys module:
//
// - sys._stats_on()
// - sys._stats_off()
// - sys._stats_clear()
// - sys._stats_dump()
//
// Python must be built with ./configure --enable-pystats to define the
// Py_STATS macro.
//
// Define _PY_INTERPRETER macro to increment interpreter_increfs and
// interpreter_decrefs. Otherwise, increment increfs and decrefs.
#ifndef Py_CPYTHON_PYSTATS_H
# error "this header file must not be included directly"
#endif
#define SPECIALIZATION_FAILURE_KINDS 36
/* Stats for determining who is calling PyEval_EvalFrame */
#define EVAL_CALL_TOTAL 0
#define EVAL_CALL_VECTOR 1
#define EVAL_CALL_GENERATOR 2
#define EVAL_CALL_LEGACY 3
#define EVAL_CALL_FUNCTION_VECTORCALL 4
#define EVAL_CALL_BUILD_CLASS 5
#define EVAL_CALL_SLOT 6
#define EVAL_CALL_FUNCTION_EX 7
#define EVAL_CALL_API 8
#define EVAL_CALL_METHOD 9
#define EVAL_CALL_KINDS 10
typedef struct _specialization_stats {
uint64_t success;
uint64_t failure;
uint64_t hit;
uint64_t deferred;
uint64_t miss;
uint64_t deopt;
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
} SpecializationStats;
typedef struct _opcode_stats {
SpecializationStats specialization;
uint64_t execution_count;
uint64_t pair_count[256];
} OpcodeStats;
typedef struct _call_stats {
uint64_t inlined_py_calls;
uint64_t pyeval_calls;
uint64_t frames_pushed;
uint64_t frame_objects_created;
uint64_t eval_calls[EVAL_CALL_KINDS];
} CallStats;
typedef struct _object_stats {
uint64_t increfs;
uint64_t decrefs;
uint64_t interpreter_increfs;
uint64_t interpreter_decrefs;
uint64_t allocations;
uint64_t allocations512;
uint64_t allocations4k;
uint64_t allocations_big;
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
uint64_t dict_dematerialized;
uint64_t type_cache_hits;
uint64_t type_cache_misses;
uint64_t type_cache_dunder_hits;
uint64_t type_cache_dunder_misses;
uint64_t type_cache_collisions;
uint64_t optimization_attempts;
uint64_t optimization_traces_created;
uint64_t optimization_traces_executed;
uint64_t optimization_uops_executed;
/* Temporary value used during GC */
uint64_t object_visits;
} ObjectStats;
typedef struct _gc_stats {
uint64_t collections;
uint64_t object_visits;
uint64_t objects_collected;
} GCStats;
typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
GCStats *gc_stats;
} PyStats;
// Export for shared extensions like 'math'
PyAPI_DATA(PyStats*) _Py_stats;
#ifdef _PY_INTERPRETER
# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0)
# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0)
#else
# define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0)
# define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0)
#endif

18
Include/internal/pycore_code.h
View File

@ -268,17 +268,17 @@ extern int _PyStaticCode_Init(PyCodeObject *co);
#ifdef Py_STATS
#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0)
#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0)
#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0)
#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0)
#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0)
#define OBJECT_STAT_INC_COND(name, cond) \
do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0)
#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0)
#define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)
#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0)
do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0)
#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0)
// Export for '_opcode' shared extension
PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);

21
Include/internal/pycore_pystats.h Normal file
View File

@ -0,0 +1,21 @@
#ifndef Py_INTERNAL_PYSTATS_H
#define Py_INTERNAL_PYSTATS_H
#ifdef __cplusplus
extern "C" {
#endif
#ifndef Py_BUILD_CORE
# error "this header requires Py_BUILD_CORE define"
#endif
#ifdef Py_STATS
extern void _Py_StatsOn(void);
extern void _Py_StatsOff(void);
extern void _Py_StatsClear(void);
extern int _Py_PrintSpecializationStats(int to_file);
#endif
#ifdef __cplusplus
}
#endif
#endif // !Py_INTERNAL_PYSTATS_H

124
Include/pystats.h
View File

@ -1,4 +1,9 @@
// Statistics on Python performance (public API).
//
// Define _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
// and Py_DECREF().
//
// See Include/cpython/pystats.h for the full API.
#ifndef Py_PYSTATS_H
#define Py_PYSTATS_H
@ -6,119 +11,16 @@
extern "C" {
#endif
#ifdef Py_STATS
#define SPECIALIZATION_FAILURE_KINDS 36
/* Stats for determining who is calling PyEval_EvalFrame */
#define EVAL_CALL_TOTAL 0
#define EVAL_CALL_VECTOR 1
#define EVAL_CALL_GENERATOR 2
#define EVAL_CALL_LEGACY 3
#define EVAL_CALL_FUNCTION_VECTORCALL 4
#define EVAL_CALL_BUILD_CLASS 5
#define EVAL_CALL_SLOT 6
#define EVAL_CALL_FUNCTION_EX 7
#define EVAL_CALL_API 8
#define EVAL_CALL_METHOD 9
#define EVAL_CALL_KINDS 10
typedef struct _specialization_stats {
uint64_t success;
uint64_t failure;
uint64_t hit;
uint64_t deferred;
uint64_t miss;
uint64_t deopt;
uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
} SpecializationStats;
typedef struct _opcode_stats {
SpecializationStats specialization;
uint64_t execution_count;
uint64_t pair_count[256];
} OpcodeStats;
typedef struct _call_stats {
uint64_t inlined_py_calls;
uint64_t pyeval_calls;
uint64_t frames_pushed;
uint64_t frame_objects_created;
uint64_t eval_calls[EVAL_CALL_KINDS];
} CallStats;
typedef struct _object_stats {
uint64_t increfs;
uint64_t decrefs;
uint64_t interpreter_increfs;
uint64_t interpreter_decrefs;
uint64_t allocations;
uint64_t allocations512;
uint64_t allocations4k;
uint64_t allocations_big;
uint64_t frees;
uint64_t to_freelist;
uint64_t from_freelist;
uint64_t new_values;
uint64_t dict_materialized_on_request;
uint64_t dict_materialized_new_key;
uint64_t dict_materialized_too_big;
uint64_t dict_materialized_str_subclass;
uint64_t dict_dematerialized;
uint64_t type_cache_hits;
uint64_t type_cache_misses;
uint64_t type_cache_dunder_hits;
uint64_t type_cache_dunder_misses;
uint64_t type_cache_collisions;
uint64_t optimization_attempts;
uint64_t optimization_traces_created;
uint64_t optimization_traces_executed;
uint64_t optimization_uops_executed;
/* Temporary value used during GC */
uint64_t object_visits;
} ObjectStats;
typedef struct _gc_stats {
uint64_t collections;
uint64_t object_visits;
uint64_t objects_collected;
} GCStats;
typedef struct _stats {
OpcodeStats opcode_stats[256];
CallStats call_stats;
ObjectStats object_stats;
GCStats *gc_stats;
} PyStats;
PyAPI_DATA(PyStats) _py_stats_struct;
PyAPI_DATA(PyStats *) _py_stats;
extern void _Py_StatsClear(void);
extern void _Py_PrintSpecializationStats(int to_file);
#ifdef _PY_INTERPRETER
#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0)
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0)
#if defined(Py_STATS) && !defined(Py_LIMITED_API)
# define Py_CPYTHON_PYSTATS_H
# include "cpython/pystats.h"
# undef Py_CPYTHON_PYSTATS_H
#else
#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0)
#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0)
#endif
#else
#define _Py_INCREF_STAT_INC() ((void)0)
#define _Py_DECREF_STAT_INC() ((void)0)
# define _Py_INCREF_STAT_INC() ((void)0)
# define _Py_DECREF_STAT_INC() ((void)0)
#endif // !Py_STATS
#ifdef __cplusplus
}
#endif
#endif /* !Py_PYSTATs_H */
#endif // !Py_PYSTATS_H

9
Lib/test/test_embed.py
View File

@ -26,6 +26,7 @@ MACOS = (sys.platform == 'darwin')
PYMEM_ALLOCATOR_NOT_SET = 0
PYMEM_ALLOCATOR_DEBUG = 2
PYMEM_ALLOCATOR_MALLOC = 3
Py_STATS = hasattr(sys, '_stats_on')
# _PyCoreConfig_InitCompatConfig()
API_COMPAT = 1
@ -512,6 +513,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'safe_path': 0,
'_is_python_build': IGNORE_CONFIG,
}
if Py_STATS:
CONFIG_COMPAT['_pystats'] = 0
if MS_WINDOWS:
CONFIG_COMPAT.update({
'legacy_windows_stdio': 0,
@ -895,6 +898,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'check_hash_pycs_mode': 'always',
'pathconfig_warnings': 0,
}
if Py_STATS:
config['_pystats'] = 1
self.check_all_configs("test_init_from_config", config, preconfig,
api=API_COMPAT)
@ -927,6 +932,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'safe_path': 1,
'int_max_str_digits': 4567,
}
if Py_STATS:
config['_pystats'] = 1
self.check_all_configs("test_init_compat_env", config, preconfig,
api=API_COMPAT)
@ -960,6 +967,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
'safe_path': 1,
'int_max_str_digits': 4567,
}
if Py_STATS:
config['_pystats'] = 1
self.check_all_configs("test_init_python_env", config, preconfig,
api=API_PYTHON)

9
Lib/test/test_sys.py
View File

@ -1200,6 +1200,15 @@ class SysModuleTest(unittest.TestCase):
get_objects = sys.getobjects(3, MyType)
self.assertEqual(len(get_objects), 3)
@unittest.skipUnless(hasattr(sys, '_stats_on'), 'need Py_STATS build')
def test_pystats(self):
# Call the functions, just check that they don't crash
# Cannot save/restore state.
sys._stats_on()
sys._stats_off()
sys._stats_clear()
sys._stats_dump()
@test.support.cpython_only
class UnraisableHookTest(unittest.TestCase):

2
Makefile.pre.in
View File

@ -1724,6 +1724,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/cpython/pylifecycle.h \
$(srcdir)/Include/cpython/pymem.h \
$(srcdir)/Include/cpython/pystate.h \
$(srcdir)/Include/cpython/pystats.h \
$(srcdir)/Include/cpython/pythonrun.h \
$(srcdir)/Include/cpython/pythread.h \
$(srcdir)/Include/cpython/setobject.h \
@ -1798,6 +1799,7 @@ PYTHON_HEADERS= \
$(srcdir)/Include/internal/pycore_pymem.h \
$(srcdir)/Include/internal/pycore_pymem_init.h \
$(srcdir)/Include/internal/pycore_pystate.h \
$(srcdir)/Include/internal/pycore_pystats.h \
$(srcdir)/Include/internal/pycore_pythonrun.h \
$(srcdir)/Include/internal/pycore_pythread.h \
$(srcdir)/Include/internal/pycore_range.h \

10
Modules/gcmodule.c
View File

@ -1200,8 +1200,8 @@ gc_collect_main(PyThreadState *tstate, int generation,
{
GC_STAT_ADD(generation, collections, 1);
#ifdef Py_STATS
if (_py_stats) {
_py_stats->object_stats.object_visits = 0;
if (_Py_stats) {
_Py_stats->object_stats.object_visits = 0;
}
#endif
int i;
@ -1362,10 +1362,10 @@ gc_collect_main(PyThreadState *tstate, int generation,
GC_STAT_ADD(generation, objects_collected, m);
#ifdef Py_STATS
if (_py_stats) {
if (_Py_stats) {
GC_STAT_ADD(generation, object_visits,
_py_stats->object_stats.object_visits);
_py_stats->object_stats.object_visits = 0;
_Py_stats->object_stats.object_visits);
_Py_stats->object_stats.object_visits = 0;
}
#endif

2
PCbuild/pythoncore.vcxproj
View File

@ -176,6 +176,7 @@
<ClInclude Include="..\Include\cpython\pylifecycle.h" />
<ClInclude Include="..\Include\cpython\pymem.h" />
<ClInclude Include="..\Include\cpython\pystate.h" />
<ClInclude Include="..\Include\cpython\pystats.h" />
<ClInclude Include="..\Include\cpython\pythonrun.h" />
<ClInclude Include="..\Include\cpython\pythread.h" />
<ClInclude Include="..\Include\cpython\setobject.h" />
@ -261,6 +262,7 @@
<ClInclude Include="..\Include\internal\pycore_pymem.h" />
<ClInclude Include="..\Include\internal\pycore_pymem_init.h" />
<ClInclude Include="..\Include\internal\pycore_pystate.h" />
<ClInclude Include="..\Include\internal\pycore_pystats.h" />
<ClInclude Include="..\Include\internal\pycore_pythonrun.h" />
<ClInclude Include="..\Include\internal\pycore_pythread.h" />
<ClInclude Include="..\Include\internal\pycore_range.h" />

6
PCbuild/pythoncore.vcxproj.filters
View File

@ -492,6 +492,9 @@
<ClInclude Include="..\Include\cpython\pystate.h">
<Filter>Include\cpython</Filter>
</ClInclude>
<ClInclude Include="..\Include\cpython\pystats.h">
<Filter>Include\cpython</Filter>
</ClInclude>
<ClInclude Include="..\Include\cpython\initconfig.h">
<Filter>Include\cpython</Filter>
</ClInclude>
@ -693,6 +696,9 @@
<ClInclude Include="..\Include\internal\pycore_pystate.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_pystats.h">
<Filter>Include\internal</Filter>
</ClInclude>
<ClInclude Include="..\Include\internal\pycore_pythonrun.h">
<Filter>Include\internal</Filter>
</ClInclude>

7
Programs/_testembed.c
View File

@ -708,6 +708,10 @@ static int test_init_from_config(void)
config.pathconfig_warnings = 0;
config.safe_path = 1;
#ifdef Py_STATS
putenv("PYTHONSTATS=");
config._pystats = 1;
#endif
putenv("PYTHONINTMAXSTRDIGITS=6666");
config.int_max_str_digits = 31337;
@ -778,6 +782,9 @@ static void set_most_env_vars(void)
putenv("PYTHONPLATLIBDIR=env_platlibdir");
putenv("PYTHONSAFEPATH=1");
putenv("PYTHONINTMAXSTRDIGITS=4567");
#ifdef Py_STATS
putenv("PYTHONSTATS=1");
#endif
}

5
Python/ceval_gil.c
View File

@ -2,11 +2,12 @@
#include "Python.h"
#include "pycore_atomic.h" // _Py_atomic_int
#include "pycore_ceval.h" // _PyEval_SignalReceived()
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
#include "pycore_pylifecycle.h" // _PyErr_Print()
#include "pycore_initconfig.h" // _PyStatus_OK()
#include "pycore_interp.h" // _Py_RunGC()
#include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
#include "pycore_pylifecycle.h" // _PyErr_Print()
#include "pycore_pymem.h" // _PyMem_IsPtrFreed()
#include "pycore_pystats.h" // _Py_PrintSpecializationStats()
/*
Notes about the implementation:

2
Python/ceval_macros.h
View File

@ -64,7 +64,7 @@
do { \
frame->prev_instr = next_instr++; \
OPCODE_EXE_INC(op); \
if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \
if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \
lastopcode = op; \
} while (0)
#else

24
Python/clinic/sysmodule.c.h generated
View File

@ -1120,7 +1120,7 @@ PyDoc_STRVAR(sys__stats_on__doc__,
"_stats_on($module, /)\n"
"--\n"
"\n"
"Turns on stats gathering (stats gathering is on by default).");
"Turns on stats gathering (stats gathering is off by default).");
#define SYS__STATS_ON_METHODDEF \
{"_stats_on", (PyCFunction)sys__stats_on, METH_NOARGS, sys__stats_on__doc__},
@ -1142,7 +1142,7 @@ PyDoc_STRVAR(sys__stats_off__doc__,
"_stats_off($module, /)\n"
"--\n"
"\n"
"Turns off stats gathering (stats gathering is on by default).");
"Turns off stats gathering (stats gathering is off by default).");
#define SYS__STATS_OFF_METHODDEF \
{"_stats_off", (PyCFunction)sys__stats_off, METH_NOARGS, sys__stats_off__doc__},
@ -1186,18 +1186,30 @@ PyDoc_STRVAR(sys__stats_dump__doc__,
"_stats_dump($module, /)\n"
"--\n"
"\n"
"Dump stats to file, and clears the stats.");
"Dump stats to file, and clears the stats.\n"
"\n"
"Return False if no statistics were not dumped because stats gathering was off.");
#define SYS__STATS_DUMP_METHODDEF \
{"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__},
static PyObject *
static int
sys__stats_dump_impl(PyObject *module);
static PyObject *
sys__stats_dump(PyObject *module, PyObject *Py_UNUSED(ignored))
{
return sys__stats_dump_impl(module);
PyObject *return_value = NULL;
int _return_value;
_return_value = sys__stats_dump_impl(module);
if ((_return_value == -1) && PyErr_Occurred()) {
goto exit;
}
return_value = PyBool_FromLong((long)_return_value);
exit:
return return_value;
}
#endif /* defined(Py_STATS) */
@ -1411,4 +1423,4 @@ exit:
#ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
#define SYS_GETANDROIDAPILEVEL_METHODDEF
#endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
/*[clinic end generated code: output=6619682ea70e7375 input=a9049054013a1b77]*/
/*[clinic end generated code: output=549bb1f92a15f916 input=a9049054013a1b77]*/

34
Python/initconfig.c
View File

@ -9,6 +9,7 @@
#include "pycore_pylifecycle.h" // _Py_PreInitializeFromConfig()
#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()
#include "pycore_pystate.h" // _PyThreadState_GET()
#include "pycore_pystats.h" // _Py_StatsOn()
#include "osdefs.h" // DELIM
@ -186,7 +187,11 @@ static const char usage_envvars[] =
"PYTHONSAFEPATH : don't prepend a potentially unsafe path to sys.path (-P)\n"
"PYTHONUNBUFFERED : disable stdout/stderr buffering (-u)\n"
"PYTHONVERBOSE : trace import statements (-v)\n"
"PYTHONWARNINGS=arg : warning control (-W arg)\n";
"PYTHONWARNINGS=arg : warning control (-W arg)\n"
#ifdef Py_STATS
"PYTHONSTATS : turns on statistics gathering\n"
#endif
;
#if defined(MS_WINDOWS)
# define PYTHONHOMEHELP "<prefix>\\python{major}{minor}"
@ -630,6 +635,9 @@ config_check_consistency(const PyConfig *config)
assert(config->int_max_str_digits >= 0);
// config->use_frozen_modules is initialized later
// by _PyConfig_InitImportConfig().
#ifdef Py_STATS
assert(config->_pystats >= 0);
#endif
return 1;
}
#endif
@ -951,6 +959,9 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
COPY_WSTRLIST(orig_argv);
COPY_ATTR(_is_python_build);
COPY_ATTR(int_max_str_digits);
#ifdef Py_STATS
COPY_ATTR(_pystats);
#endif
#undef COPY_ATTR
#undef COPY_WSTR_ATTR
@ -1058,6 +1069,9 @@ _PyConfig_AsDict(const PyConfig *config)
SET_ITEM_INT(safe_path);
SET_ITEM_INT(_is_python_build);
SET_ITEM_INT(int_max_str_digits);
#ifdef Py_STATS
SET_ITEM_INT(_pystats);
#endif
return dict;
@ -1365,6 +1379,9 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
GET_UINT(safe_path);
GET_UINT(_is_python_build);
GET_INT(int_max_str_digits);
#ifdef Py_STATS
GET_UINT(_pystats);
#endif
#undef CHECK_VALUE
#undef GET_UINT
@ -2116,7 +2133,13 @@ config_read(PyConfig *config, int compute_path_config)
#ifdef Py_STATS
if (config_get_xoption(config, L"pystats")) {
_py_stats = &_py_stats_struct;
config->_pystats = 1;
}
else if (config_get_env(config, "PYTHONSTATS")) {
config->_pystats = 1;
}
if (config->_pystats < 0) {
config->_pystats = 0;
}
#endif
@ -2254,6 +2277,13 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime)
{
return _PyStatus_NO_MEMORY();
}
#ifdef Py_STATS
if (config->_pystats) {
_Py_StatsOn();
}
#endif
return _PyStatus_OK();
}

100
Python/specialize.c
View File

@ -13,17 +13,17 @@
#include "pycore_pylifecycle.h" // _PyOS_URandomNonblock()
#include "pycore_runtime.h" // _Py_ID()
#include <stdlib.h> // rand()
/* For guidance on adding or extending families of instructions see
* ./adaptive.md
*/
#ifdef Py_STATS
GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 };
PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] };
PyStats *_py_stats = NULL;
static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats };
PyStats *_Py_stats = NULL;
#define ADD_STAT_TO_DICT(res, field) \
do { \
@ -83,7 +83,7 @@ add_stat_dict(
int opcode,
const char *name) {
SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization;
SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization;
PyObject *d = stats_to_dict(stats);
if (d == NULL) {
return -1;
@ -93,7 +93,6 @@ add_stat_dict(
return err;
}
#ifdef Py_STATS
PyObject*
_Py_GetSpecializationStats(void) {
PyObject *stats = PyDict_New();
@ -120,7 +119,6 @@ _Py_GetSpecializationStats(void) {
}
return stats;
}
#endif
#define PRINT_STAT(i, field) \
@ -218,7 +216,8 @@ print_gc_stats(FILE *out, GCStats *stats)
}
static void
print_stats(FILE *out, PyStats *stats) {
print_stats(FILE *out, PyStats *stats)
{
print_spec_stats(out, stats->opcode_stats);
print_call_stats(out, &stats->call_stats);
print_object_stats(out, &stats->object_stats);
@ -226,18 +225,56 @@ print_stats(FILE *out, PyStats *stats) {
}
void
_Py_StatsClear(void)
_Py_StatsOn(void)
{
for (int i = 0; i < NUM_GENERATIONS; i++) {
_py_gc_stats[i] = (GCStats) { 0 };
}
_py_stats_struct = (PyStats) { 0 };
_py_stats_struct.gc_stats = _py_gc_stats;
_Py_stats = &_Py_stats_struct;
}
void
_Py_StatsOff(void)
{
_Py_stats = NULL;
}
void
_Py_StatsClear(void)
{
memset(&_py_gc_stats, 0, sizeof(_py_gc_stats));
memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct));
_Py_stats_struct.gc_stats = _py_gc_stats;
}
static int
mem_is_zero(unsigned char *ptr, size_t size)
{
for (size_t i=0; i < size; i++) {
if (*ptr != 0) {
return 0;
}
ptr++;
}
return 1;
}
int
_Py_PrintSpecializationStats(int to_file)
{
PyStats *stats = &_Py_stats_struct;
#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA)))
int is_zero = (
MEM_IS_ZERO(stats->gc_stats) // is a pointer
&& MEM_IS_ZERO(&stats->opcode_stats)
&& MEM_IS_ZERO(&stats->call_stats)
&& MEM_IS_ZERO(&stats->object_stats)
);
#undef MEM_IS_ZERO
if (is_zero) {
// gh-108753: -X pystats command line was used, but then _stats_off()
// and _stats_clear() have been called: in this case, avoid printing
// useless "all zeros" statistics.
return 0;
}
FILE *out = stderr;
if (to_file) {
/* Write to a file instead of stderr. */
@ -268,26 +305,25 @@ _Py_PrintSpecializationStats(int to_file)
else {
fprintf(out, "Specialization stats:\n");
}
print_stats(out, &_py_stats_struct);
print_stats(out, stats);
if (out != stderr) {
fclose(out);
}
return 1;
}
#ifdef Py_STATS
#define SPECIALIZATION_FAIL(opcode, kind) \
do { \
if (_py_stats) { \
_py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
if (_Py_stats) { \
_Py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
} \
} while (0)
#endif
#endif
#endif // Py_STATS
#ifndef SPECIALIZATION_FAIL
#define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
# define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
#endif
// Initialize warmup counters and insert superinstructions. This cannot fail.
@ -1067,7 +1103,7 @@ load_attr_fail_kind(DescriptorClassification kind)
}
Py_UNREACHABLE();
}
#endif
#endif // Py_STATS
static int
specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
@ -1306,7 +1342,7 @@ binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
}
return SPEC_FAIL_OTHER;
}
#endif
#endif // Py_STATS
static int
function_kind(PyCodeObject *code) {
@ -1545,7 +1581,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins
}
goto fail;
}
#endif
#endif // Py_STATS
SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
fail:
STAT_INC(STORE_SUBSCR, failure);
@ -1690,7 +1726,7 @@ meth_descr_call_fail_kind(int ml_flags)
return SPEC_FAIL_CALL_BAD_CALL_FLAGS;
}
}
#endif
#endif // Py_STATS
static int
specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
@ -1871,7 +1907,7 @@ call_fail_kind(PyObject *callable)
}
return SPEC_FAIL_OTHER;
}
#endif
#endif // Py_STATS
/* TODO:
@ -1995,7 +2031,7 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
}
Py_UNREACHABLE();
}
#endif
#endif // Py_STATS
void
_Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
@ -2102,7 +2138,7 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs)
}
return SPEC_FAIL_OTHER;
}
#endif
#endif // Py_STATS
void
_Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
@ -2165,7 +2201,7 @@ unpack_sequence_fail_kind(PyObject *seq)
}
return SPEC_FAIL_OTHER;
}
#endif
#endif // Py_STATS
void
_Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
@ -2206,7 +2242,6 @@ success:
}
#ifdef Py_STATS
int
_PySpecialization_ClassifyIterator(PyObject *iter)
{
@ -2277,8 +2312,7 @@ int
}
return SPEC_FAIL_OTHER;
}
#endif
#endif // Py_STATS
void
_Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg)
@ -2431,7 +2465,7 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr)
goto failure;
}
SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OTHER);
#endif
#endif // Py_STATS
failure:
STAT_INC(TO_BOOL, failure);
instr->op.code = TO_BOOL;

28
Python/sysmodule.c
View File

@ -30,6 +30,7 @@ Data members:
#include "pycore_pymath.h" // _PY_SHORT_FLOAT_REPR
#include "pycore_pymem.h" // _PyMem_SetDefaultAllocator()
#include "pycore_pystate.h" // _PyThreadState_GET()
#include "pycore_pystats.h" // _Py_PrintSpecializationStats()
#include "pycore_structseq.h" // _PyStructSequence_InitBuiltinWithFlags()
#include "pycore_sysmodule.h" // export _PySys_GetSizeOf()
#include "pycore_tuple.h" // _PyTuple_FromArray()
@ -2106,32 +2107,33 @@ sys_is_finalizing_impl(PyObject *module)
return PyBool_FromLong(Py_IsFinalizing());
}
#ifdef Py_STATS
/*[clinic input]
sys._stats_on
Turns on stats gathering (stats gathering is on by default).
Turns on stats gathering (stats gathering is off by default).
[clinic start generated code]*/
static PyObject *
sys__stats_on_impl(PyObject *module)
/*[clinic end generated code: output=aca53eafcbb4d9fe input=8ddc6df94e484f3a]*/
/*[clinic end generated code: output=aca53eafcbb4d9fe input=43b5bfe145299e55]*/
{
_py_stats = &_py_stats_struct;
_Py_StatsOn();
Py_RETURN_NONE;
}
/*[clinic input]
sys._stats_off
Turns off stats gathering (stats gathering is on by default).
Turns off stats gathering (stats gathering is off by default).
[clinic start generated code]*/
static PyObject *
sys__stats_off_impl(PyObject *module)
/*[clinic end generated code: output=1534c1ee63812214 input=b3e50e71ecf29f66]*/
/*[clinic end generated code: output=1534c1ee63812214 input=d1a84c60c56cbce2]*/
{
_py_stats = NULL;
_Py_StatsOff();
Py_RETURN_NONE;
}
@ -2150,21 +2152,23 @@ sys__stats_clear_impl(PyObject *module)
}
/*[clinic input]
sys._stats_dump
sys._stats_dump -> bool
Dump stats to file, and clears the stats.
Return False if no statistics were not dumped because stats gathering was off.
[clinic start generated code]*/
static PyObject *
static int
sys__stats_dump_impl(PyObject *module)
/*[clinic end generated code: output=79f796fb2b4ddf05 input=92346f16d64f6f95]*/
/*[clinic end generated code: output=6e346b4ba0de4489 input=31a489e39418b2a5]*/
{
_Py_PrintSpecializationStats(1);
int res = _Py_PrintSpecializationStats(1);
_Py_StatsClear();
Py_RETURN_NONE;
return res;
}
#endif // Py_STATS
#endif
#ifdef ANDROID_API_LEVEL
/*[clinic input]