gh-108753: Enhance pystats (#108754)

Statistics gathering is now off by default. Use the "-X pystats"
command line option or set the new PYTHONSTATS environment variable
to 1 to turn statistics gathering on at Python startup.

Statistics are no longer dumped at exit if statistics gathering was
off or statistics have been cleared.

Changes:

* Add PYTHONSTATS environment variable.
* sys._stats_dump() now returns False if statistics are not dumped
  because they are all equal to zero.
* Add PyConfig._pystats member.
* Add tests on sys functions and on setting PyConfig._pystats to 1.
* Add Include/cpython/pystats.h and Include/internal/pycore_pystats.h
  header files.
* Rename '_py_stats' variable to '_Py_stats'.
* Exclude Include/cpython/pystats.h from the Py_LIMITED_API.
* Move pystats.h include from object.h to Python.h.
* Add _Py_StatsOn() and _Py_StatsOff() functions. Remove
  '_py_stats_struct' variable from the API: make it static in
  specialize.c.
* Document API in Include/pystats.h and Include/cpython/pystats.h.
* Complete pystats documentation in Doc/using/configure.rst.
* Don't write "all zeros" stats: if _stats_off() and _stats_clear()
  or _stats_dump() were called.
* _PyEval_Fini() now always call _Py_PrintSpecializationStats() which
  does nothing if stats are all zeros.

Co-authored-by: Michael Droettboom <mdboom@gmail.com>

Doc/using/configure.rst

@@ -192,14 +192,69 @@ General Options
 
 .. cmdoption:: --enable-pystats
 
-   Turn on internal statistics gathering.
+   Turn on internal Python performance statistics gathering.
+
+   By default, statistics gathering is off. Use ``python3 -X pystats`` command
+   or set ``PYTHONSTATS=1`` environment variable to turn on statistics
+   gathering at Python startup.
+
+   At Python exit, dump statistics if statistics gathering was on and not
+   cleared.
+
+   Effects:
+
+   * Add :option:`-X pystats <-X>` command line option.
+   * Add :envvar:`!PYTHONSTATS` environment variable.
+   * Define the ``Py_STATS`` macro.
+   * Add functions to the :mod:`sys` module:
+
+     * :func:`!sys._stats_on`: Turns on statistics gathering.
+     * :func:`!sys._stats_off`: Turns off statistics gathering.
+     * :func:`!sys._stats_clear`: Clears the statistics.
+     * :func:`!sys._stats_dump`: Dump statistics to file, and clears the statistics.
 
    The statistics will be dumped to a arbitrary (probably unique) file in
-   ``/tmp/py_stats/``, or ``C:\temp\py_stats\`` on Windows. If that directory
+   ``/tmp/py_stats/`` (Unix) or ``C:\temp\py_stats\`` (Windows). If that
-   does not exist, results will be printed on stdout.
+   directory does not exist, results will be printed on stderr.
 
    Use ``Tools/scripts/summarize_stats.py`` to read the stats.
 
+   Statistics:
+
+   * Opcode:
+
+     * Specialization: success, failure, hit, deferred, miss, deopt, failures;
+     * Execution count;
+     * Pair count.
+
+   * Call:
+
+     * Inlined Python calls;
+     * PyEval calls;
+     * Frames pushed;
+     * Frame object created;
+     * Eval calls: vector, generator, legacy, function VECTORCALL, build class,
+       slot, function "ex", API, method.
+
+   * Object:
+
+     * incref and decref;
+     * interpreter incref and decref;
+     * allocations: all, 512 bytes, 4 kiB, big;
+     * free;
+     * to/from free lists;
+     * dictionary materialized/dematerialized;
+     * type cache;
+     * optimization attemps;
+     * optimization traces created/executed;
+     * uops executed.
+
+   * Garbage collector:
+
+     * Garbage collections;
+     * Objects visited;
+     * Objects collected.
+
    .. versionadded:: 3.11
 
 .. cmdoption:: --disable-gil

Include/cpython/initconfig.h

@@ -215,6 +215,11 @@ typedef struct PyConfig {
 
     // If non-zero, we believe we're running from a source tree.
     int _is_python_build;
+
+#ifdef Py_STATS
+    // If non-zero, turns on statistics gathering.
+    int _pystats;
+#endif
 } PyConfig;
 
 PyAPI_FUNC(void) PyConfig_InitPythonConfig(PyConfig *config);

Include/cpython/pystats.h

@@ -0,0 +1,120 @@
+// Statistics on Python performance.
+//
+// API:
+//
+// - _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
+//   and Py_DECREF().
+// - _Py_stats variable
+//
+// Functions of the sys module:
+//
+// - sys._stats_on()
+// - sys._stats_off()
+// - sys._stats_clear()
+// - sys._stats_dump()
+//
+// Python must be built with ./configure --enable-pystats to define the
+// Py_STATS macro.
+//
+// Define _PY_INTERPRETER macro to increment interpreter_increfs and
+// interpreter_decrefs. Otherwise, increment increfs and decrefs.
+
+#ifndef Py_CPYTHON_PYSTATS_H
+#  error "this header file must not be included directly"
+#endif
+
+#define SPECIALIZATION_FAILURE_KINDS 36
+
+/* Stats for determining who is calling PyEval_EvalFrame */
+#define EVAL_CALL_TOTAL 0
+#define EVAL_CALL_VECTOR 1
+#define EVAL_CALL_GENERATOR 2
+#define EVAL_CALL_LEGACY 3
+#define EVAL_CALL_FUNCTION_VECTORCALL 4
+#define EVAL_CALL_BUILD_CLASS 5
+#define EVAL_CALL_SLOT 6
+#define EVAL_CALL_FUNCTION_EX 7
+#define EVAL_CALL_API 8
+#define EVAL_CALL_METHOD 9
+
+#define EVAL_CALL_KINDS 10
+
+typedef struct _specialization_stats {
+    uint64_t success;
+    uint64_t failure;
+    uint64_t hit;
+    uint64_t deferred;
+    uint64_t miss;
+    uint64_t deopt;
+    uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
+} SpecializationStats;
+
+typedef struct _opcode_stats {
+    SpecializationStats specialization;
+    uint64_t execution_count;
+    uint64_t pair_count[256];
+} OpcodeStats;
+
+typedef struct _call_stats {
+    uint64_t inlined_py_calls;
+    uint64_t pyeval_calls;
+    uint64_t frames_pushed;
+    uint64_t frame_objects_created;
+    uint64_t eval_calls[EVAL_CALL_KINDS];
+} CallStats;
+
+typedef struct _object_stats {
+    uint64_t increfs;
+    uint64_t decrefs;
+    uint64_t interpreter_increfs;
+    uint64_t interpreter_decrefs;
+    uint64_t allocations;
+    uint64_t allocations512;
+    uint64_t allocations4k;
+    uint64_t allocations_big;
+    uint64_t frees;
+    uint64_t to_freelist;
+    uint64_t from_freelist;
+    uint64_t new_values;
+    uint64_t dict_materialized_on_request;
+    uint64_t dict_materialized_new_key;
+    uint64_t dict_materialized_too_big;
+    uint64_t dict_materialized_str_subclass;
+    uint64_t dict_dematerialized;
+    uint64_t type_cache_hits;
+    uint64_t type_cache_misses;
+    uint64_t type_cache_dunder_hits;
+    uint64_t type_cache_dunder_misses;
+    uint64_t type_cache_collisions;
+    uint64_t optimization_attempts;
+    uint64_t optimization_traces_created;
+    uint64_t optimization_traces_executed;
+    uint64_t optimization_uops_executed;
+    /* Temporary value used during GC */
+    uint64_t object_visits;
+} ObjectStats;
+
+typedef struct _gc_stats {
+    uint64_t collections;
+    uint64_t object_visits;
+    uint64_t objects_collected;
+} GCStats;
+
+typedef struct _stats {
+    OpcodeStats opcode_stats[256];
+    CallStats call_stats;
+    ObjectStats object_stats;
+    GCStats *gc_stats;
+} PyStats;
+
+
+// Export for shared extensions like 'math'
+PyAPI_DATA(PyStats*) _Py_stats;
+
+#ifdef _PY_INTERPRETER
+#  define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_increfs++; } while (0)
+#  define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.interpreter_decrefs++; } while (0)
+#else
+#  define _Py_INCREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.increfs++; } while (0)
+#  define _Py_DECREF_STAT_INC() do { if (_Py_stats) _Py_stats->object_stats.decrefs++; } while (0)
+#endif

Include/internal/pycore_code.h

@@ -268,17 +268,17 @@ extern int _PyStaticCode_Init(PyCodeObject *co);
 
 #ifdef Py_STATS
 
-#define STAT_INC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name++; } while (0)
+#define STAT_INC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name++; } while (0)
-#define STAT_DEC(opname, name) do { if (_py_stats) _py_stats->opcode_stats[opname].specialization.name--; } while (0)
+#define STAT_DEC(opname, name) do { if (_Py_stats) _Py_stats->opcode_stats[opname].specialization.name--; } while (0)
-#define OPCODE_EXE_INC(opname) do { if (_py_stats) _py_stats->opcode_stats[opname].execution_count++; } while (0)
+#define OPCODE_EXE_INC(opname) do { if (_Py_stats) _Py_stats->opcode_stats[opname].execution_count++; } while (0)
-#define CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.name++; } while (0)
+#define CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.name++; } while (0)
-#define OBJECT_STAT_INC(name) do { if (_py_stats) _py_stats->object_stats.name++; } while (0)
+#define OBJECT_STAT_INC(name) do { if (_Py_stats) _Py_stats->object_stats.name++; } while (0)
 #define OBJECT_STAT_INC_COND(name, cond) \
-    do { if (_py_stats && cond) _py_stats->object_stats.name++; } while (0)
+    do { if (_Py_stats && cond) _Py_stats->object_stats.name++; } while (0)
-#define EVAL_CALL_STAT_INC(name) do { if (_py_stats) _py_stats->call_stats.eval_calls[name]++; } while (0)
+#define EVAL_CALL_STAT_INC(name) do { if (_Py_stats) _Py_stats->call_stats.eval_calls[name]++; } while (0)
 #define EVAL_CALL_STAT_INC_IF_FUNCTION(name, callable) \
-    do { if (_py_stats && PyFunction_Check(callable)) _py_stats->call_stats.eval_calls[name]++; } while (0)
+    do { if (_Py_stats && PyFunction_Check(callable)) _Py_stats->call_stats.eval_calls[name]++; } while (0)
-#define GC_STAT_ADD(gen, name, n) do { if (_py_stats) _py_stats->gc_stats[(gen)].name += (n); } while (0)
+#define GC_STAT_ADD(gen, name, n) do { if (_Py_stats) _Py_stats->gc_stats[(gen)].name += (n); } while (0)
 
 // Export for '_opcode' shared extension
 PyAPI_FUNC(PyObject*) _Py_GetSpecializationStats(void);

Include/internal/pycore_pystats.h

@@ -0,0 +1,21 @@
+#ifndef Py_INTERNAL_PYSTATS_H
+#define Py_INTERNAL_PYSTATS_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef Py_STATS
+extern void _Py_StatsOn(void);
+extern void _Py_StatsOff(void);
+extern void _Py_StatsClear(void);
+extern int _Py_PrintSpecializationStats(int to_file);
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif  // !Py_INTERNAL_PYSTATS_H

Include/pystats.h

@@ -1,4 +1,9 @@
-
+// Statistics on Python performance (public API).
+//
+// Define _Py_INCREF_STAT_INC() and _Py_DECREF_STAT_INC() used by Py_INCREF()
+// and Py_DECREF().
+//
+// See Include/cpython/pystats.h for the full API.
 
 #ifndef Py_PYSTATS_H
 #define Py_PYSTATS_H
@@ -6,119 +11,16 @@
 extern "C" {
 #endif
 
-#ifdef Py_STATS
+#if defined(Py_STATS) && !defined(Py_LIMITED_API)
-
+#  define Py_CPYTHON_PYSTATS_H
-#define SPECIALIZATION_FAILURE_KINDS 36
+#  include "cpython/pystats.h"
-
+#  undef Py_CPYTHON_PYSTATS_H
-/* Stats for determining who is calling PyEval_EvalFrame */
-#define EVAL_CALL_TOTAL 0
-#define EVAL_CALL_VECTOR 1
-#define EVAL_CALL_GENERATOR 2
-#define EVAL_CALL_LEGACY 3
-#define EVAL_CALL_FUNCTION_VECTORCALL 4
-#define EVAL_CALL_BUILD_CLASS 5
-#define EVAL_CALL_SLOT 6
-#define EVAL_CALL_FUNCTION_EX 7
-#define EVAL_CALL_API 8
-#define EVAL_CALL_METHOD 9
-
-#define EVAL_CALL_KINDS 10
-
-typedef struct _specialization_stats {
-    uint64_t success;
-    uint64_t failure;
-    uint64_t hit;
-    uint64_t deferred;
-    uint64_t miss;
-    uint64_t deopt;
-    uint64_t failure_kinds[SPECIALIZATION_FAILURE_KINDS];
-} SpecializationStats;
-
-typedef struct _opcode_stats {
-    SpecializationStats specialization;
-    uint64_t execution_count;
-    uint64_t pair_count[256];
-} OpcodeStats;
-
-typedef struct _call_stats {
-    uint64_t inlined_py_calls;
-    uint64_t pyeval_calls;
-    uint64_t frames_pushed;
-    uint64_t frame_objects_created;
-    uint64_t eval_calls[EVAL_CALL_KINDS];
-} CallStats;
-
-typedef struct _object_stats {
-    uint64_t increfs;
-    uint64_t decrefs;
-    uint64_t interpreter_increfs;
-    uint64_t interpreter_decrefs;
-    uint64_t allocations;
-    uint64_t allocations512;
-    uint64_t allocations4k;
-    uint64_t allocations_big;
-    uint64_t frees;
-    uint64_t to_freelist;
-    uint64_t from_freelist;
-    uint64_t new_values;
-    uint64_t dict_materialized_on_request;
-    uint64_t dict_materialized_new_key;
-    uint64_t dict_materialized_too_big;
-    uint64_t dict_materialized_str_subclass;
-    uint64_t dict_dematerialized;
-    uint64_t type_cache_hits;
-    uint64_t type_cache_misses;
-    uint64_t type_cache_dunder_hits;
-    uint64_t type_cache_dunder_misses;
-    uint64_t type_cache_collisions;
-    uint64_t optimization_attempts;
-    uint64_t optimization_traces_created;
-    uint64_t optimization_traces_executed;
-    uint64_t optimization_uops_executed;
-    /* Temporary value used during GC */
-    uint64_t object_visits;
-} ObjectStats;
-
-typedef struct _gc_stats {
-    uint64_t collections;
-    uint64_t object_visits;
-    uint64_t objects_collected;
-} GCStats;
-
-typedef struct _stats {
-    OpcodeStats opcode_stats[256];
-    CallStats call_stats;
-    ObjectStats object_stats;
-    GCStats *gc_stats;
-} PyStats;
-
-
-PyAPI_DATA(PyStats) _py_stats_struct;
-PyAPI_DATA(PyStats *) _py_stats;
-
-extern void _Py_StatsClear(void);
-extern void _Py_PrintSpecializationStats(int to_file);
-
-#ifdef _PY_INTERPRETER
-
-#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_increfs++; } while (0)
-#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.interpreter_decrefs++; } while (0)
-
 #else
-
+#  define _Py_INCREF_STAT_INC() ((void)0)
-#define _Py_INCREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.increfs++; } while (0)
+#  define _Py_DECREF_STAT_INC() ((void)0)
-#define _Py_DECREF_STAT_INC() do { if (_py_stats) _py_stats->object_stats.decrefs++; } while (0)
-
-#endif
-
-#else
-
-#define _Py_INCREF_STAT_INC() ((void)0)
-#define _Py_DECREF_STAT_INC() ((void)0)
-
 #endif  // !Py_STATS
 
 #ifdef __cplusplus
 }
 #endif
-#endif /* !Py_PYSTATs_H */
+#endif   // !Py_PYSTATS_H

Lib/test/test_embed.py

@@ -26,6 +26,7 @@ MACOS = (sys.platform == 'darwin')
 PYMEM_ALLOCATOR_NOT_SET = 0
 PYMEM_ALLOCATOR_DEBUG = 2
 PYMEM_ALLOCATOR_MALLOC = 3
+Py_STATS = hasattr(sys, '_stats_on')
 
 # _PyCoreConfig_InitCompatConfig()
 API_COMPAT = 1
@@ -512,6 +513,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
         'safe_path': 0,
         '_is_python_build': IGNORE_CONFIG,
     }
+    if Py_STATS:
+        CONFIG_COMPAT['_pystats'] = 0
     if MS_WINDOWS:
         CONFIG_COMPAT.update({
             'legacy_windows_stdio': 0,
@@ -895,6 +898,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'check_hash_pycs_mode': 'always',
             'pathconfig_warnings': 0,
         }
+        if Py_STATS:
+            config['_pystats'] = 1
         self.check_all_configs("test_init_from_config", config, preconfig,
                                api=API_COMPAT)
 
@@ -927,6 +932,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'safe_path': 1,
             'int_max_str_digits': 4567,
         }
+        if Py_STATS:
+            config['_pystats'] = 1
         self.check_all_configs("test_init_compat_env", config, preconfig,
                                api=API_COMPAT)
 
@@ -960,6 +967,8 @@ class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
             'safe_path': 1,
             'int_max_str_digits': 4567,
         }
+        if Py_STATS:
+            config['_pystats'] = 1
         self.check_all_configs("test_init_python_env", config, preconfig,
                                api=API_PYTHON)
 

Lib/test/test_sys.py

@@ -1200,6 +1200,15 @@ class SysModuleTest(unittest.TestCase):
         get_objects = sys.getobjects(3, MyType)
         self.assertEqual(len(get_objects), 3)
 
+    @unittest.skipUnless(hasattr(sys, '_stats_on'), 'need Py_STATS build')
+    def test_pystats(self):
+        # Call the functions, just check that they don't crash
+        # Cannot save/restore state.
+        sys._stats_on()
+        sys._stats_off()
+        sys._stats_clear()
+        sys._stats_dump()
+
 
 @test.support.cpython_only
 class UnraisableHookTest(unittest.TestCase):

Makefile.pre.in

@@ -1724,6 +1724,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/cpython/pylifecycle.h \
 		$(srcdir)/Include/cpython/pymem.h \
 		$(srcdir)/Include/cpython/pystate.h \
+		$(srcdir)/Include/cpython/pystats.h \
 		$(srcdir)/Include/cpython/pythonrun.h \
 		$(srcdir)/Include/cpython/pythread.h \
 		$(srcdir)/Include/cpython/setobject.h \
@@ -1798,6 +1799,7 @@ PYTHON_HEADERS= \
 		$(srcdir)/Include/internal/pycore_pymem.h \
 		$(srcdir)/Include/internal/pycore_pymem_init.h \
 		$(srcdir)/Include/internal/pycore_pystate.h \
+		$(srcdir)/Include/internal/pycore_pystats.h \
 		$(srcdir)/Include/internal/pycore_pythonrun.h \
 		$(srcdir)/Include/internal/pycore_pythread.h \
 		$(srcdir)/Include/internal/pycore_range.h \

Modules/gcmodule.c

@@ -1200,8 +1200,8 @@ gc_collect_main(PyThreadState *tstate, int generation,
 {
     GC_STAT_ADD(generation, collections, 1);
 #ifdef Py_STATS
-    if (_py_stats) {
+    if (_Py_stats) {
-        _py_stats->object_stats.object_visits = 0;
+        _Py_stats->object_stats.object_visits = 0;
     }
 #endif
     int i;
@@ -1362,10 +1362,10 @@ gc_collect_main(PyThreadState *tstate, int generation,
 
     GC_STAT_ADD(generation, objects_collected, m);
 #ifdef Py_STATS
-    if (_py_stats) {
+    if (_Py_stats) {
         GC_STAT_ADD(generation, object_visits,
-            _py_stats->object_stats.object_visits);
+            _Py_stats->object_stats.object_visits);
-        _py_stats->object_stats.object_visits = 0;
+        _Py_stats->object_stats.object_visits = 0;
     }
 #endif
 

PCbuild/pythoncore.vcxproj

@@ -176,6 +176,7 @@
     <ClInclude Include="..\Include\cpython\pylifecycle.h" />
     <ClInclude Include="..\Include\cpython\pymem.h" />
     <ClInclude Include="..\Include\cpython\pystate.h" />
+    <ClInclude Include="..\Include\cpython\pystats.h" />
     <ClInclude Include="..\Include\cpython\pythonrun.h" />
     <ClInclude Include="..\Include\cpython\pythread.h" />
     <ClInclude Include="..\Include\cpython\setobject.h" />
@@ -261,6 +262,7 @@
     <ClInclude Include="..\Include\internal\pycore_pymem.h" />
     <ClInclude Include="..\Include\internal\pycore_pymem_init.h" />
     <ClInclude Include="..\Include\internal\pycore_pystate.h" />
+    <ClInclude Include="..\Include\internal\pycore_pystats.h" />
     <ClInclude Include="..\Include\internal\pycore_pythonrun.h" />
     <ClInclude Include="..\Include\internal\pycore_pythread.h" />
     <ClInclude Include="..\Include\internal\pycore_range.h" />

PCbuild/pythoncore.vcxproj.filters

@@ -492,6 +492,9 @@
     <ClInclude Include="..\Include\cpython\pystate.h">
       <Filter>Include\cpython</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\cpython\pystats.h">
+      <Filter>Include\cpython</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\cpython\initconfig.h">
       <Filter>Include\cpython</Filter>
     </ClInclude>
@@ -693,6 +696,9 @@
     <ClInclude Include="..\Include\internal\pycore_pystate.h">
       <Filter>Include\internal</Filter>
     </ClInclude>
+    <ClInclude Include="..\Include\internal\pycore_pystats.h">
+      <Filter>Include\internal</Filter>
+    </ClInclude>
     <ClInclude Include="..\Include\internal\pycore_pythonrun.h">
       <Filter>Include\internal</Filter>
     </ClInclude>

Programs/_testembed.c

@@ -708,6 +708,10 @@ static int test_init_from_config(void)
     config.pathconfig_warnings = 0;
 
     config.safe_path = 1;
+#ifdef Py_STATS
+    putenv("PYTHONSTATS=");
+    config._pystats = 1;
+#endif
 
     putenv("PYTHONINTMAXSTRDIGITS=6666");
     config.int_max_str_digits = 31337;
@@ -778,6 +782,9 @@ static void set_most_env_vars(void)
     putenv("PYTHONPLATLIBDIR=env_platlibdir");
     putenv("PYTHONSAFEPATH=1");
     putenv("PYTHONINTMAXSTRDIGITS=4567");
+#ifdef Py_STATS
+    putenv("PYTHONSTATS=1");
+#endif
 }
 
 

Python/ceval_gil.c

@@ -2,11 +2,12 @@
 #include "Python.h"
 #include "pycore_atomic.h"        // _Py_atomic_int
 #include "pycore_ceval.h"         // _PyEval_SignalReceived()
-#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
-#include "pycore_pylifecycle.h"   // _PyErr_Print()
 #include "pycore_initconfig.h"    // _PyStatus_OK()
 #include "pycore_interp.h"        // _Py_RunGC()
+#include "pycore_pyerrors.h"      // _PyErr_GetRaisedException()
+#include "pycore_pylifecycle.h"   // _PyErr_Print()
 #include "pycore_pymem.h"         // _PyMem_IsPtrFreed()
+#include "pycore_pystats.h"       // _Py_PrintSpecializationStats()
 
 /*
    Notes about the implementation:

Python/ceval_macros.h

@@ -64,7 +64,7 @@
     do { \
         frame->prev_instr = next_instr++; \
         OPCODE_EXE_INC(op); \
-        if (_py_stats) _py_stats->opcode_stats[lastopcode].pair_count[op]++; \
+        if (_Py_stats) _Py_stats->opcode_stats[lastopcode].pair_count[op]++; \
         lastopcode = op; \
     } while (0)
 #else

Python/clinic/sysmodule.c.h

@@ -1120,7 +1120,7 @@ PyDoc_STRVAR(sys__stats_on__doc__,
 "_stats_on($module, /)\n"
 "--\n"
 "\n"
-"Turns on stats gathering (stats gathering is on by default).");
+"Turns on stats gathering (stats gathering is off by default).");
 
 #define SYS__STATS_ON_METHODDEF    \
     {"_stats_on", (PyCFunction)sys__stats_on, METH_NOARGS, sys__stats_on__doc__},
@@ -1142,7 +1142,7 @@ PyDoc_STRVAR(sys__stats_off__doc__,
 "_stats_off($module, /)\n"
 "--\n"
 "\n"
-"Turns off stats gathering (stats gathering is on by default).");
+"Turns off stats gathering (stats gathering is off by default).");
 
 #define SYS__STATS_OFF_METHODDEF    \
     {"_stats_off", (PyCFunction)sys__stats_off, METH_NOARGS, sys__stats_off__doc__},
@@ -1186,18 +1186,30 @@ PyDoc_STRVAR(sys__stats_dump__doc__,
 "_stats_dump($module, /)\n"
 "--\n"
 "\n"
-"Dump stats to file, and clears the stats.");
+"Dump stats to file, and clears the stats.\n"
+"\n"
+"Return False if no statistics were not dumped because stats gathering was off.");
 
 #define SYS__STATS_DUMP_METHODDEF    \
     {"_stats_dump", (PyCFunction)sys__stats_dump, METH_NOARGS, sys__stats_dump__doc__},
 
-static PyObject *
+static int
 sys__stats_dump_impl(PyObject *module);
 
 static PyObject *
 sys__stats_dump(PyObject *module, PyObject *Py_UNUSED(ignored))
 {
-    return sys__stats_dump_impl(module);
+    PyObject *return_value = NULL;
+    int _return_value;
+
+    _return_value = sys__stats_dump_impl(module);
+    if ((_return_value == -1) && PyErr_Occurred()) {
+        goto exit;
+    }
+    return_value = PyBool_FromLong((long)_return_value);
+
+exit:
+    return return_value;
 }
 
 #endif /* defined(Py_STATS) */
@@ -1411,4 +1423,4 @@ exit:
 #ifndef SYS_GETANDROIDAPILEVEL_METHODDEF
     #define SYS_GETANDROIDAPILEVEL_METHODDEF
 #endif /* !defined(SYS_GETANDROIDAPILEVEL_METHODDEF) */
-/*[clinic end generated code: output=6619682ea70e7375 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=549bb1f92a15f916 input=a9049054013a1b77]*/

Python/initconfig.c

@@ -9,6 +9,7 @@
 #include "pycore_pylifecycle.h"   // _Py_PreInitializeFromConfig()
 #include "pycore_pymem.h"         // _PyMem_SetDefaultAllocator()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_pystats.h"       // _Py_StatsOn()
 
 #include "osdefs.h"               // DELIM
 
@@ -186,7 +187,11 @@ static const char usage_envvars[] =
 "PYTHONSAFEPATH          : don't prepend a potentially unsafe path to sys.path (-P)\n"
 "PYTHONUNBUFFERED        : disable stdout/stderr buffering (-u)\n"
 "PYTHONVERBOSE           : trace import statements (-v)\n"
-"PYTHONWARNINGS=arg      : warning control (-W arg)\n";
+"PYTHONWARNINGS=arg      : warning control (-W arg)\n"
+#ifdef Py_STATS
+"PYTHONSTATS             : turns on statistics gathering\n"
+#endif
+;
 
 #if defined(MS_WINDOWS)
 #  define PYTHONHOMEHELP "<prefix>\\python{major}{minor}"
@@ -630,6 +635,9 @@ config_check_consistency(const PyConfig *config)
     assert(config->int_max_str_digits >= 0);
     // config->use_frozen_modules is initialized later
     // by _PyConfig_InitImportConfig().
+#ifdef Py_STATS
+    assert(config->_pystats >= 0);
+#endif
     return 1;
 }
 #endif
@@ -951,6 +959,9 @@ _PyConfig_Copy(PyConfig *config, const PyConfig *config2)
     COPY_WSTRLIST(orig_argv);
     COPY_ATTR(_is_python_build);
     COPY_ATTR(int_max_str_digits);
+#ifdef Py_STATS
+    COPY_ATTR(_pystats);
+#endif
 
 #undef COPY_ATTR
 #undef COPY_WSTR_ATTR
@@ -1058,6 +1069,9 @@ _PyConfig_AsDict(const PyConfig *config)
     SET_ITEM_INT(safe_path);
     SET_ITEM_INT(_is_python_build);
     SET_ITEM_INT(int_max_str_digits);
+#ifdef Py_STATS
+    SET_ITEM_INT(_pystats);
+#endif
 
     return dict;
 
@@ -1365,6 +1379,9 @@ _PyConfig_FromDict(PyConfig *config, PyObject *dict)
     GET_UINT(safe_path);
     GET_UINT(_is_python_build);
     GET_INT(int_max_str_digits);
+#ifdef Py_STATS
+    GET_UINT(_pystats);
+#endif
 
 #undef CHECK_VALUE
 #undef GET_UINT
@@ -2116,7 +2133,13 @@ config_read(PyConfig *config, int compute_path_config)
 
 #ifdef Py_STATS
     if (config_get_xoption(config, L"pystats")) {
-        _py_stats = &_py_stats_struct;
+        config->_pystats = 1;
+    }
+    else if (config_get_env(config, "PYTHONSTATS")) {
+        config->_pystats = 1;
+    }
+    if (config->_pystats < 0) {
+        config->_pystats = 0;
     }
 #endif
 
@@ -2254,6 +2277,13 @@ _PyConfig_Write(const PyConfig *config, _PyRuntimeState *runtime)
     {
         return _PyStatus_NO_MEMORY();
     }
+
+#ifdef Py_STATS
+    if (config->_pystats) {
+        _Py_StatsOn();
+    }
+#endif
+
     return _PyStatus_OK();
 }
 

Python/specialize.c

@@ -13,17 +13,17 @@
 #include "pycore_pylifecycle.h"   // _PyOS_URandomNonblock()
 #include "pycore_runtime.h"       // _Py_ID()
 
-
 #include <stdlib.h> // rand()
 
+
 /* For guidance on adding or extending families of instructions see
  * ./adaptive.md
  */
 
 #ifdef Py_STATS
 GCStats _py_gc_stats[NUM_GENERATIONS] = { 0 };
-PyStats _py_stats_struct = { .gc_stats = &_py_gc_stats[0] };
+static PyStats _Py_stats_struct = { .gc_stats = _py_gc_stats };
-PyStats *_py_stats = NULL;
+PyStats *_Py_stats = NULL;
 
 #define ADD_STAT_TO_DICT(res, field) \
     do { \
@@ -83,7 +83,7 @@ add_stat_dict(
     int opcode,
     const char *name) {
 
-    SpecializationStats *stats = &_py_stats_struct.opcode_stats[opcode].specialization;
+    SpecializationStats *stats = &_Py_stats_struct.opcode_stats[opcode].specialization;
     PyObject *d = stats_to_dict(stats);
     if (d == NULL) {
         return -1;
@@ -93,7 +93,6 @@ add_stat_dict(
     return err;
 }
 
-#ifdef Py_STATS
 PyObject*
 _Py_GetSpecializationStats(void) {
     PyObject *stats = PyDict_New();
@@ -120,7 +119,6 @@ _Py_GetSpecializationStats(void) {
     }
     return stats;
 }
-#endif
 
 
 #define PRINT_STAT(i, field) \
@@ -218,7 +216,8 @@ print_gc_stats(FILE *out, GCStats *stats)
 }
 
 static void
-print_stats(FILE *out, PyStats *stats) {
+print_stats(FILE *out, PyStats *stats)
+{
     print_spec_stats(out, stats->opcode_stats);
     print_call_stats(out, &stats->call_stats);
     print_object_stats(out, &stats->object_stats);
@@ -226,18 +225,56 @@ print_stats(FILE *out, PyStats *stats) {
 }
 
 void
-_Py_StatsClear(void)
+_Py_StatsOn(void)
 {
-    for (int i = 0; i < NUM_GENERATIONS; i++) {
+    _Py_stats = &_Py_stats_struct;
-        _py_gc_stats[i] = (GCStats) { 0 };
-    }
-    _py_stats_struct = (PyStats) { 0 };
-    _py_stats_struct.gc_stats = _py_gc_stats;
 }
 
 void
+_Py_StatsOff(void)
+{
+    _Py_stats = NULL;
+}
+
+void
+_Py_StatsClear(void)
+{
+    memset(&_py_gc_stats, 0, sizeof(_py_gc_stats));
+    memset(&_Py_stats_struct, 0, sizeof(_Py_stats_struct));
+    _Py_stats_struct.gc_stats = _py_gc_stats;
+}
+
+static int
+mem_is_zero(unsigned char *ptr, size_t size)
+{
+    for (size_t i=0; i < size; i++) {
+        if (*ptr != 0) {
+            return 0;
+        }
+        ptr++;
+    }
+    return 1;
+}
+
+int
 _Py_PrintSpecializationStats(int to_file)
 {
+    PyStats *stats = &_Py_stats_struct;
+#define MEM_IS_ZERO(DATA) mem_is_zero((unsigned char*)DATA, sizeof(*(DATA)))
+    int is_zero = (
+        MEM_IS_ZERO(stats->gc_stats)  // is a pointer
+        && MEM_IS_ZERO(&stats->opcode_stats)
+        && MEM_IS_ZERO(&stats->call_stats)
+        && MEM_IS_ZERO(&stats->object_stats)
+    );
+#undef MEM_IS_ZERO
+    if (is_zero) {
+        // gh-108753: -X pystats command line was used, but then _stats_off()
+        // and _stats_clear() have been called: in this case, avoid printing
+        // useless "all zeros" statistics.
+        return 0;
+    }
+
     FILE *out = stderr;
     if (to_file) {
         /* Write to a file instead of stderr. */
@@ -268,26 +305,25 @@ _Py_PrintSpecializationStats(int to_file)
     else {
         fprintf(out, "Specialization stats:\n");
     }
-    print_stats(out, &_py_stats_struct);
+    print_stats(out, stats);
     if (out != stderr) {
         fclose(out);
     }
+    return 1;
 }
 
-#ifdef Py_STATS
-
 #define SPECIALIZATION_FAIL(opcode, kind) \
 do { \
-    if (_py_stats) { \
+    if (_Py_stats) { \
-        _py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
+        _Py_stats->opcode_stats[opcode].specialization.failure_kinds[kind]++; \
     } \
 } while (0)
 
-#endif
+#endif  // Py_STATS
-#endif
+
 
 #ifndef SPECIALIZATION_FAIL
-#define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
+#  define SPECIALIZATION_FAIL(opcode, kind) ((void)0)
 #endif
 
 // Initialize warmup counters and insert superinstructions. This cannot fail.
@@ -1067,7 +1103,7 @@ load_attr_fail_kind(DescriptorClassification kind)
     }
     Py_UNREACHABLE();
 }
-#endif
+#endif   // Py_STATS
 
 static int
 specialize_class_load_attr(PyObject *owner, _Py_CODEUNIT *instr,
@@ -1306,7 +1342,7 @@ binary_subscr_fail_kind(PyTypeObject *container_type, PyObject *sub)
     }
     return SPEC_FAIL_OTHER;
 }
-#endif
+#endif   // Py_STATS
 
 static int
 function_kind(PyCodeObject *code) {
@@ -1545,7 +1581,7 @@ _Py_Specialize_StoreSubscr(PyObject *container, PyObject *sub, _Py_CODEUNIT *ins
         }
         goto fail;
     }
-#endif
+#endif   // Py_STATS
     SPECIALIZATION_FAIL(STORE_SUBSCR, SPEC_FAIL_OTHER);
 fail:
     STAT_INC(STORE_SUBSCR, failure);
@@ -1690,7 +1726,7 @@ meth_descr_call_fail_kind(int ml_flags)
             return SPEC_FAIL_CALL_BAD_CALL_FLAGS;
     }
 }
-#endif
+#endif   // Py_STATS
 
 static int
 specialize_method_descriptor(PyMethodDescrObject *descr, _Py_CODEUNIT *instr,
@@ -1871,7 +1907,7 @@ call_fail_kind(PyObject *callable)
     }
     return SPEC_FAIL_OTHER;
 }
-#endif
+#endif   // Py_STATS
 
 
 /* TODO:
@@ -1995,7 +2031,7 @@ binary_op_fail_kind(int oparg, PyObject *lhs, PyObject *rhs)
     }
     Py_UNREACHABLE();
 }
-#endif
+#endif   // Py_STATS
 
 void
 _Py_Specialize_BinaryOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
@@ -2102,7 +2138,7 @@ compare_op_fail_kind(PyObject *lhs, PyObject *rhs)
     }
     return SPEC_FAIL_OTHER;
 }
-#endif
+#endif   // Py_STATS
 
 void
 _Py_Specialize_CompareOp(PyObject *lhs, PyObject *rhs, _Py_CODEUNIT *instr,
@@ -2165,7 +2201,7 @@ unpack_sequence_fail_kind(PyObject *seq)
     }
     return SPEC_FAIL_OTHER;
 }
-#endif
+#endif   // Py_STATS
 
 void
 _Py_Specialize_UnpackSequence(PyObject *seq, _Py_CODEUNIT *instr, int oparg)
@@ -2206,7 +2242,6 @@ success:
 }
 
 #ifdef Py_STATS
-
 int
  _PySpecialization_ClassifyIterator(PyObject *iter)
 {
@@ -2277,8 +2312,7 @@ int
     }
     return SPEC_FAIL_OTHER;
 }
-
+#endif   // Py_STATS
-#endif
 
 void
 _Py_Specialize_ForIter(PyObject *iter, _Py_CODEUNIT *instr, int oparg)
@@ -2431,7 +2465,7 @@ _Py_Specialize_ToBool(PyObject *value, _Py_CODEUNIT *instr)
         goto failure;
     }
     SPECIALIZATION_FAIL(TO_BOOL, SPEC_FAIL_OTHER);
-#endif
+#endif   // Py_STATS
 failure:
     STAT_INC(TO_BOOL, failure);
     instr->op.code = TO_BOOL;

Python/sysmodule.c

@@ -30,6 +30,7 @@ Data members:
 #include "pycore_pymath.h"        // _PY_SHORT_FLOAT_REPR
 #include "pycore_pymem.h"         // _PyMem_SetDefaultAllocator()
 #include "pycore_pystate.h"       // _PyThreadState_GET()
+#include "pycore_pystats.h"       // _Py_PrintSpecializationStats()
 #include "pycore_structseq.h"     // _PyStructSequence_InitBuiltinWithFlags()
 #include "pycore_sysmodule.h"     // export _PySys_GetSizeOf()
 #include "pycore_tuple.h"         // _PyTuple_FromArray()
@@ -2106,32 +2107,33 @@ sys_is_finalizing_impl(PyObject *module)
     return PyBool_FromLong(Py_IsFinalizing());
 }
 
+
 #ifdef Py_STATS
 /*[clinic input]
 sys._stats_on
 
-Turns on stats gathering (stats gathering is on by default).
+Turns on stats gathering (stats gathering is off by default).
 [clinic start generated code]*/
 
 static PyObject *
 sys__stats_on_impl(PyObject *module)
-/*[clinic end generated code: output=aca53eafcbb4d9fe input=8ddc6df94e484f3a]*/
+/*[clinic end generated code: output=aca53eafcbb4d9fe input=43b5bfe145299e55]*/
 {
-    _py_stats = &_py_stats_struct;
+    _Py_StatsOn();
     Py_RETURN_NONE;
 }
 
 /*[clinic input]
 sys._stats_off
 
-Turns off stats gathering (stats gathering is on by default).
+Turns off stats gathering (stats gathering is off by default).
 [clinic start generated code]*/
 
 static PyObject *
 sys__stats_off_impl(PyObject *module)
-/*[clinic end generated code: output=1534c1ee63812214 input=b3e50e71ecf29f66]*/
+/*[clinic end generated code: output=1534c1ee63812214 input=d1a84c60c56cbce2]*/
 {
-    _py_stats = NULL;
+    _Py_StatsOff();
     Py_RETURN_NONE;
 }
 
@@ -2150,21 +2152,23 @@ sys__stats_clear_impl(PyObject *module)
 }
 
 /*[clinic input]
-sys._stats_dump
+sys._stats_dump -> bool
 
 Dump stats to file, and clears the stats.
+
+Return False if no statistics were not dumped because stats gathering was off.
 [clinic start generated code]*/
 
-static PyObject *
+static int
 sys__stats_dump_impl(PyObject *module)
-/*[clinic end generated code: output=79f796fb2b4ddf05 input=92346f16d64f6f95]*/
+/*[clinic end generated code: output=6e346b4ba0de4489 input=31a489e39418b2a5]*/
 {
-    _Py_PrintSpecializationStats(1);
+    int res = _Py_PrintSpecializationStats(1);
     _Py_StatsClear();
-    Py_RETURN_NONE;
+    return res;
 }
+#endif   // Py_STATS
 
-#endif
 
 #ifdef ANDROID_API_LEVEL
 /*[clinic input]