From 91a8e002c21a5388c5152c5a4871b9a2d85f0fc1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 7 Dec 2022 15:56:31 -0700 Subject: [PATCH] gh-81057: Move More Globals to _PyRuntimeState (gh-100092) https://github.com/python/cpython/issues/81057 --- Include/internal/pycore_dtoa.h | 11 ++++++---- Include/internal/pycore_obmalloc.h | 1 + Include/internal/pycore_obmalloc_init.h | 1 + Include/internal/pycore_parser.h | 20 +++++++++++++++++ Include/internal/pycore_runtime.h | 5 +++++ Include/internal/pycore_unicodeobject.h | 3 +++ Modules/_io/bufferedio.c | 24 ++++++++++----------- Objects/obmalloc.c | 3 ++- Objects/unicodeobject.c | 8 ++++--- Parser/pegen.c | 4 ++-- Python/dtoa.c | 8 ++----- Python/initconfig.c | 14 ++++++------ Tools/c-analyzer/cpython/globals-to-fix.tsv | 9 -------- Tools/c-analyzer/cpython/ignored.tsv | 11 ---------- 14 files changed, 66 insertions(+), 56 deletions(-) diff --git a/Include/internal/pycore_dtoa.h b/Include/internal/pycore_dtoa.h index fdc6e74ecd2..67189cf0ade 100644 --- a/Include/internal/pycore_dtoa.h +++ b/Include/internal/pycore_dtoa.h @@ -25,7 +25,7 @@ Bigint { #ifdef Py_USING_MEMORY_DEBUGGER struct _dtoa_runtime_state { - int _not_used; + int _not_used; }; #define _dtoa_runtime_state_INIT {0} @@ -41,9 +41,12 @@ struct _dtoa_runtime_state { ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double)) struct _dtoa_runtime_state { - struct Bigint *freelist[Bigint_Kmax+1]; - double preallocated[Bigint_PREALLOC_SIZE]; - double *preallocated_next; + /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ + // XXX This should be freed during runtime fini. + struct Bigint *p5s; + struct Bigint *freelist[Bigint_Kmax+1]; + double preallocated[Bigint_PREALLOC_SIZE]; + double *preallocated_next; }; #define _dtoa_runtime_state_INIT(runtime) \ { \ diff --git a/Include/internal/pycore_obmalloc.h b/Include/internal/pycore_obmalloc.h index 93349d89c6a..a5c7f4528f9 100644 --- a/Include/internal/pycore_obmalloc.h +++ b/Include/internal/pycore_obmalloc.h @@ -658,6 +658,7 @@ struct _obmalloc_usage { struct _obmalloc_state { + int dump_debug_stats; struct _obmalloc_pools pools; struct _obmalloc_mgmt mgmt; struct _obmalloc_usage usage; diff --git a/Include/internal/pycore_obmalloc_init.h b/Include/internal/pycore_obmalloc_init.h index c0fb057d066..c9f197e72de 100644 --- a/Include/internal/pycore_obmalloc_init.h +++ b/Include/internal/pycore_obmalloc_init.h @@ -56,6 +56,7 @@ extern "C" { #define _obmalloc_state_INIT(obmalloc) \ { \ + .dump_debug_stats = -1, \ .pools = { \ .used = _obmalloc_pools_INIT(obmalloc.pools), \ }, \ diff --git a/Include/internal/pycore_parser.h b/Include/internal/pycore_parser.h index e2de24e2ca9..2d2b56bd824 100644 --- a/Include/internal/pycore_parser.h +++ b/Include/internal/pycore_parser.h @@ -8,12 +8,31 @@ extern "C" { # error "this header requires Py_BUILD_CORE define" #endif + +#include "pycore_pyarena.h" // PyArena + + +#ifdef Py_DEBUG +#define _PYPEGEN_NSTATISTICS 2000 +#endif + +struct _parser_runtime_state { +#ifdef Py_DEBUG + long memo_statistics[_PYPEGEN_NSTATISTICS]; +#else + int _not_used; +#endif +}; + + + extern struct _mod* _PyParser_ASTFromString( const char *str, PyObject* filename, int mode, PyCompilerFlags *flags, PyArena *arena); + extern struct _mod* _PyParser_ASTFromFile( FILE *fp, PyObject *filename_ob, @@ -25,6 +44,7 @@ extern struct _mod* _PyParser_ASTFromFile( int *errcode, PyArena *arena); + #ifdef __cplusplus } #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index c1829cb1bda..0720e2ed442 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -17,6 +17,7 @@ extern "C" { #include "pycore_global_objects.h" // struct _Py_global_objects #include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState +#include "pycore_parser.h" // struct _parser_runtime_state #include "pycore_pymem.h" // struct _pymem_allocators #include "pycore_pyhash.h" // struct pyhash_runtime_state #include "pycore_obmalloc.h" // struct obmalloc_state @@ -129,6 +130,10 @@ typedef struct pyruntimestate { unsigned long main_thread; + PyWideStringList orig_argv; + + struct _parser_runtime_state parser; + #define NEXITFUNCS 32 void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; diff --git a/Include/internal/pycore_unicodeobject.h b/Include/internal/pycore_unicodeobject.h index b315ca1ae5b..19faceebf1d 100644 --- a/Include/internal/pycore_unicodeobject.h +++ b/Include/internal/pycore_unicodeobject.h @@ -9,6 +9,7 @@ extern "C" { #endif #include "pycore_fileutils.h" // _Py_error_handler +#include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI void _PyUnicode_ExactDealloc(PyObject *op); @@ -52,6 +53,8 @@ struct _Py_unicode_ids { struct _Py_unicode_state { struct _Py_unicode_fs_codec fs_codec; + _PyUnicode_Name_CAPI *ucnhash_capi; + // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId() struct _Py_unicode_ids ids; }; diff --git a/Modules/_io/bufferedio.c b/Modules/_io/bufferedio.c index 6df55b5b830..ba8969f0bcd 100644 --- a/Modules/_io/bufferedio.c +++ b/Modules/_io/bufferedio.c @@ -746,26 +746,26 @@ _buffered_init(buffered *self) int _PyIO_trap_eintr(void) { - static PyObject *eintr_int = NULL; PyObject *typ, *val, *tb; PyOSErrorObject *env_err; - - if (eintr_int == NULL) { - eintr_int = PyLong_FromLong(EINTR); - assert(eintr_int != NULL); - } if (!PyErr_ExceptionMatches(PyExc_OSError)) return 0; PyErr_Fetch(&typ, &val, &tb); PyErr_NormalizeException(&typ, &val, &tb); env_err = (PyOSErrorObject *) val; assert(env_err != NULL); - if (env_err->myerrno != NULL && - PyObject_RichCompareBool(env_err->myerrno, eintr_int, Py_EQ) > 0) { - Py_DECREF(typ); - Py_DECREF(val); - Py_XDECREF(tb); - return 1; + if (env_err->myerrno != NULL) { + assert(EINTR > 0 && EINTR < INT_MAX); + assert(PyLong_CheckExact(env_err->myerrno)); + int overflow; + int myerrno = PyLong_AsLongAndOverflow(env_err->myerrno, &overflow); + PyErr_Clear(); + if (myerrno == EINTR) { + Py_DECREF(typ); + Py_DECREF(val); + Py_XDECREF(tb); + return 1; + } } /* This silences any error set by PyObject_RichCompareBool() */ PyErr_Restore(typ, val, tb); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 4c08bc214cd..276c5a276c0 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -908,11 +908,12 @@ new_arena(void) struct arena_object* arenaobj; uint excess; /* number of bytes above pool alignment */ void *address; - static int debug_stats = -1; + int debug_stats = _PyRuntime.obmalloc.dump_debug_stats; if (debug_stats == -1) { const char *opt = Py_GETENV("PYTHONMALLOCSTATS"); debug_stats = (opt != NULL && *opt != '\0'); + _PyRuntime.obmalloc.dump_debug_stats = debug_stats; } if (debug_stats) { _PyObject_DebugMallocStats(stderr); diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index deeca35714b..b721ccd805e 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -5697,8 +5697,6 @@ PyUnicode_AsUTF16String(PyObject *unicode) /* --- Unicode Escape Codec ----------------------------------------------- */ -static _PyUnicode_Name_CAPI *ucnhash_capi = NULL; - PyObject * _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, Py_ssize_t size, @@ -5711,6 +5709,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, const char *end; PyObject *errorHandler = NULL; PyObject *exc = NULL; + _PyUnicode_Name_CAPI *ucnhash_capi; + PyInterpreterState *interp = _PyInterpreterState_Get(); // so we can remember if we've seen an invalid escape char or not *first_invalid_escape = NULL; @@ -5858,6 +5858,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, /* \N{name} */ case 'N': + ucnhash_capi = interp->unicode.ucnhash_capi; if (ucnhash_capi == NULL) { /* load the unicode data module */ ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import( @@ -5869,6 +5870,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s, ); goto onError; } + interp->unicode.ucnhash_capi = ucnhash_capi; } message = "malformed \\N character escape"; @@ -15128,10 +15130,10 @@ _PyUnicode_Fini(PyInterpreterState *interp) assert(get_interned_dict() == NULL); // bpo-47182: force a unicodedata CAPI capsule re-import on // subsequent initialization of main interpreter. - ucnhash_capi = NULL; } _PyUnicode_FiniEncodings(&state->fs_codec); + interp->unicode.ucnhash_capi = NULL; unicode_clear_identifiers(state); } diff --git a/Parser/pegen.c b/Parser/pegen.c index d34a86e9c88..d84e06861ed 100644 --- a/Parser/pegen.c +++ b/Parser/pegen.c @@ -246,8 +246,8 @@ _PyPegen_fill_token(Parser *p) // The array counts the number of tokens skipped by memoization, // indexed by type. -#define NSTATISTICS 2000 -static long memo_statistics[NSTATISTICS]; +#define NSTATISTICS _PYPEGEN_NSTATISTICS +#define memo_statistics _PyRuntime.parser.memo_statistics void _PyPegen_clear_memo_statistics() diff --git a/Python/dtoa.c b/Python/dtoa.c index 1b47d83bf77..cff5f1b0658 100644 --- a/Python/dtoa.c +++ b/Python/dtoa.c @@ -673,10 +673,6 @@ mult(Bigint *a, Bigint *b) #ifndef Py_USING_MEMORY_DEBUGGER -/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */ - -static Bigint *p5s; - /* multiply the Bigint b by 5**k. Returns a pointer to the result, or NULL on failure; if the returned pointer is distinct from b then the original Bigint b will have been Bfree'd. Ignores the sign of b. */ @@ -696,7 +692,7 @@ pow5mult(Bigint *b, int k) if (!(k >>= 2)) return b; - p5 = p5s; + p5 = _PyRuntime.dtoa.p5s; if (!p5) { /* first time */ p5 = i2b(625); @@ -704,7 +700,7 @@ pow5mult(Bigint *b, int k) Bfree(b); return NULL; } - p5s = p5; + _PyRuntime.dtoa.p5s = p5; p5->next = 0; } for(;;) { diff --git a/Python/initconfig.c b/Python/initconfig.c index 67f6777d3b1..64ae987b3f3 100644 --- a/Python/initconfig.c +++ b/Python/initconfig.c @@ -595,17 +595,13 @@ _Py_ClearStandardStreamEncoding(void) /* --- Py_GetArgcArgv() ------------------------------------------- */ -/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */ -static PyWideStringList orig_argv = {.length = 0, .items = NULL}; - - void _Py_ClearArgcArgv(void) { PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - _PyWideStringList_Clear(&orig_argv); + _PyWideStringList_Clear(&_PyRuntime.orig_argv); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -620,7 +616,9 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - res = _PyWideStringList_Copy(&orig_argv, &argv_list); + // XXX _PyRuntime.orig_argv only gets cleared by Py_Main(), + // so it it currently leaks for embedders. + res = _PyWideStringList_Copy(&_PyRuntime.orig_argv, &argv_list); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; @@ -631,8 +629,8 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv) void Py_GetArgcArgv(int *argc, wchar_t ***argv) { - *argc = (int)orig_argv.length; - *argv = orig_argv.items; + *argc = (int)_PyRuntime.orig_argv.length; + *argv = _PyRuntime.orig_argv.items; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index af5b7eb8eaf..5c8164517f1 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -305,20 +305,12 @@ Objects/sliceobject.c - _Py_EllipsisObject - ##----------------------- ## effectively-const but initialized lazily -## idempotent -Python/dtoa.c - p5s - -Objects/obmalloc.c new_arena debug_stats - - ## others Python/perf_trampoline.c - perf_map_file - -Objects/unicodeobject.c - ucnhash_capi - ##----------------------- ## state -## local buffer -Python/suggestions.c levenshtein_distance buffer - - ## other Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - @@ -398,7 +390,6 @@ Modules/faulthandler.c - old_stack - ##----------------------- ## initialized once -Modules/_io/bufferedio.c _PyIO_trap_eintr eintr_int - Modules/posixmodule.c os_dup2_impl dup3_works - Modules/posixmodule.c - structseq_new - Modules/posixmodule.c - ticks_per_second - diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 128336a997e..257823574fa 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -87,8 +87,6 @@ Parser/myreadline.c - PyOS_ReadlineFunctionPointer - Python/initconfig.c - _Py_StandardStreamEncoding - Python/initconfig.c - _Py_StandardStreamErrors - -# XXX This only gets cleared by Py_Main(). -Python/initconfig.c - orig_argv - ##----------------------- ## public C-API @@ -143,15 +141,6 @@ Python/pylifecycle.c - runtime_initialized - Modules/syslogmodule.c - S_ident_o - Modules/syslogmodule.c - S_log_open - -##----------------------- -## *not* tied to init/fini cycle - -# These do not ge reset with each init/fini cycle. -# XXX These should probably be tied to init/fini. Move to _PyRuntimeState? - -# special-use diagnistic state -Parser/pegen.c - memo_statistics - - ##----------------------- ## one-off temporary state