diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst index c7b3ea95b08..1516e3c083f 100644 --- a/Doc/library/sys.rst +++ b/Doc/library/sys.rst @@ -106,6 +106,22 @@ always available. This function should be used for internal and specialized purposes only. +.. function:: _debugmallocstats() + + Print low-level information to stderr about the state of CPython's memory + allocator. + + If Python is configured --with-pydebug, it also performs some expensive + internal consistency checks. + + .. versionadded:: 3.3 + + .. impl-detail:: + + This function is specific to CPython. The exact output format is not + defined here, and may change. + + .. data:: dllhandle Integer specifying the handle of the Python DLL. Availability: Windows. diff --git a/Include/dictobject.h b/Include/dictobject.h index d908524f8d9..d89aac89321 100644 --- a/Include/dictobject.h +++ b/Include/dictobject.h @@ -111,6 +111,7 @@ PyAPI_FUNC(int) PyDict_DelItemString(PyObject *dp, const char *key); #ifndef Py_LIMITED_API int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff --git a/Include/floatobject.h b/Include/floatobject.h index 46ef6e60620..e240fdb8f68 100644 --- a/Include/floatobject.h +++ b/Include/floatobject.h @@ -110,6 +110,8 @@ PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le); /* free list api */ PyAPI_FUNC(int) PyFloat_ClearFreeList(void); +PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); + /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(int) _PyFloat_FormatAdvancedWriter( diff --git a/Include/frameobject.h b/Include/frameobject.h index a8df4455a83..ac2f790d13d 100644 --- a/Include/frameobject.h +++ b/Include/frameobject.h @@ -79,6 +79,8 @@ PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); PyAPI_FUNC(int) PyFrame_ClearFreeList(void); +PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); + /* Return the line of code the frame is currently executing. */ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); diff --git a/Include/listobject.h b/Include/listobject.h index 6fd374bb86e..dc62aee91a5 100644 --- a/Include/listobject.h +++ b/Include/listobject.h @@ -64,6 +64,7 @@ PyAPI_FUNC(PyObject *) PyList_AsTuple(PyObject *); PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(int) PyList_ClearFreeList(void); +PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); #endif /* Macro, trading safety for speed */ diff --git a/Include/methodobject.h b/Include/methodobject.h index 0eabbb7c9d1..3cc2ea9308f 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -82,6 +82,11 @@ typedef struct { PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyCFunction_DebugMallocStats(FILE *out); +PyAPI_FUNC(void) _PyMethod_DebugMallocStats(FILE *out); +#endif + #ifdef __cplusplus } #endif diff --git a/Include/object.h b/Include/object.h index e341532cb38..68001e79f48 100644 --- a/Include/object.h +++ b/Include/object.h @@ -977,6 +977,14 @@ PyAPI_DATA(PyObject *) _PyTrash_delete_later; else \ _PyTrash_deposit_object((PyObject*)op); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) +_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, + size_t sizeof_block); +PyAPI_FUNC(void) +_PyObject_DebugTypeStats(FILE *out); +#endif /* ifndef Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff --git a/Include/objimpl.h b/Include/objimpl.h index 8fc3fc1e9a7..b1a624c8ebf 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -101,13 +101,15 @@ PyAPI_FUNC(void) PyObject_Free(void *); /* Macros */ #ifdef WITH_PYMALLOC +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); +#endif /* #ifndef Py_LIMITED_API */ #ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes); PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes); PyAPI_FUNC(void) _PyObject_DebugFree(void *p); PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p); PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p); -PyAPI_FUNC(void) _PyObject_DebugMallocStats(void); PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes); PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes); PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p); diff --git a/Include/setobject.h b/Include/setobject.h index 00e53449295..a14874bb0f6 100644 --- a/Include/setobject.h +++ b/Include/setobject.h @@ -101,6 +101,7 @@ PyAPI_FUNC(PyObject *) PySet_Pop(PyObject *set); PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); PyAPI_FUNC(int) PySet_ClearFreeList(void); +PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff --git a/Include/tupleobject.h b/Include/tupleobject.h index f17b7882bed..c273ce7dc8b 100644 --- a/Include/tupleobject.h +++ b/Include/tupleobject.h @@ -63,6 +63,9 @@ PyAPI_FUNC(void) _PyTuple_MaybeUntrack(PyObject *); #endif PyAPI_FUNC(int) PyTuple_ClearFreeList(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); +#endif /* Py_LIMITED_API */ #ifdef __cplusplus } diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index 796a1c5db3c..9a6dd5ba698 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -603,6 +603,12 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.implementation.name, sys.implementation.name.lower()) + def test_debugmallocstats(self): + # Test sys._debugmallocstats() + from test.script_helper import assert_python_ok + args = ['-c', 'import sys; sys._debugmallocstats()'] + ret, out, err = assert_python_ok(*args) + self.assertIn(b"free PyDictObjects", err) class SizeofTest(unittest.TestCase): diff --git a/Misc/NEWS b/Misc/NEWS index af1e26c3dbe..f1c3cd2512b 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -147,6 +147,9 @@ Library - Issue #14963: Convert contextlib.ExitStack.__exit__ to use an iterative algorithm (Patch by Alon Horev) +- Issue #14785: Add sys._debugmallocstats() to help debug low-level memory + allocation issues + C-API ----- diff --git a/Objects/classobject.c b/Objects/classobject.c index 09b95bf29fc..0416a6a7330 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -400,6 +400,15 @@ PyMethod_Fini(void) (void)PyMethod_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyMethod_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyMethodObject", + numfree, sizeof(PyMethodObject)); +} + /* ------------------------------------------------------------------------ * instance method */ diff --git a/Objects/dictobject.c b/Objects/dictobject.c index fd1d46c6c32..4af5c49e957 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -255,6 +255,15 @@ PyDict_ClearFreeList(void) return ret; } +/* Print summary info about the state of the optimized allocator */ +void +_PyDict_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyDictObject", numfree, sizeof(PyDictObject)); +} + + void PyDict_Fini(void) { diff --git a/Objects/floatobject.c b/Objects/floatobject.c index 3c742c3a038..a42be71c23b 100644 --- a/Objects/floatobject.c +++ b/Objects/floatobject.c @@ -1933,6 +1933,16 @@ PyFloat_Fini(void) (void)PyFloat_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyFloat_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyFloatObject", + numfree, sizeof(PyFloatObject)); +} + + /*---------------------------------------------------------------------------- * _PyFloat_{Pack,Unpack}{4,8}. See floatobject.h. */ diff --git a/Objects/frameobject.c b/Objects/frameobject.c index 929385fe28d..808e595157b 100644 --- a/Objects/frameobject.c +++ b/Objects/frameobject.c @@ -955,3 +955,13 @@ PyFrame_Fini(void) Py_XDECREF(builtin_object); builtin_object = NULL; } + +/* Print summary info about the state of the optimized allocator */ +void +_PyFrame_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyFrameObject", + numfree, sizeof(PyFrameObject)); +} + diff --git a/Objects/listobject.c b/Objects/listobject.c index e59c9b1b635..6e0d094154e 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -117,6 +117,15 @@ PyList_Fini(void) PyList_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyList_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyListObject", + numfree, sizeof(PyListObject)); +} + PyObject * PyList_New(Py_ssize_t size) { diff --git a/Objects/methodobject.c b/Objects/methodobject.c index c3a64098dc9..1d143f91319 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -338,6 +338,15 @@ PyCFunction_Fini(void) (void)PyCFunction_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyCFunction_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyCFunction", + numfree, sizeof(PyCFunction)); +} + /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(), but it's part of the API so we need to keep a function around that existing C extensions can call. diff --git a/Objects/object.c b/Objects/object.c index 1211cc3154d..f4c0208bd3b 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -1852,6 +1852,18 @@ PyMem_Free(void *p) PyMem_FREE(p); } +void +_PyObject_DebugTypeStats(FILE *out) +{ + _PyCFunction_DebugMallocStats(out); + _PyDict_DebugMallocStats(out); + _PyFloat_DebugMallocStats(out); + _PyFrame_DebugMallocStats(out); + _PyList_DebugMallocStats(out); + _PyMethod_DebugMallocStats(out); + _PySet_DebugMallocStats(out); + _PyTuple_DebugMallocStats(out); +} /* These methods are used to control infinite recursion in repr, str, print, etc. Container objects that may recursively contain themselves, diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 9cd6a50466c..925482156de 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -523,12 +523,10 @@ static struct arena_object* usable_arenas = NULL; /* Number of arenas allocated that haven't been free()'d. */ static size_t narenas_currently_allocated = 0; -#ifdef PYMALLOC_DEBUG /* Total number of times malloc() called to allocate an arena. */ static size_t ntimes_arena_allocated = 0; /* High water mark (max value ever seen) for narenas_currently_allocated. */ static size_t narenas_highwater = 0; -#endif /* Allocate a new arena. If we run out of memory, return NULL. Else * allocate a new arena, and return the address of an arena_object @@ -545,7 +543,7 @@ new_arena(void) #ifdef PYMALLOC_DEBUG if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(); + _PyObject_DebugMallocStats(stderr); #endif if (unused_arena_objects == NULL) { uint i; @@ -613,11 +611,9 @@ new_arena(void) arenaobj->address = (uptr)address; ++narenas_currently_allocated; -#ifdef PYMALLOC_DEBUG ++ntimes_arena_allocated; if (narenas_currently_allocated > narenas_highwater) narenas_highwater = narenas_currently_allocated; -#endif arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ @@ -1723,17 +1719,19 @@ _PyObject_DebugDumpAddress(const void *p) } } +#endif /* PYMALLOC_DEBUG */ + static size_t -printone(const char* msg, size_t value) +printone(FILE *out, const char* msg, size_t value) { int i, k; char buf[100]; size_t origvalue = value; - fputs(msg, stderr); + fputs(msg, out); for (i = (int)strlen(msg); i < 35; ++i) - fputc(' ', stderr); - fputc('=', stderr); + fputc(' ', out); + fputc('=', out); /* Write the value with commas. */ i = 22; @@ -1754,17 +1752,33 @@ printone(const char* msg, size_t value) while (i >= 0) buf[i--] = ' '; - fputs(buf, stderr); + fputs(buf, out); return origvalue; } -/* Print summary info to stderr about the state of pymalloc's structures. +void +_PyDebugAllocatorStats(FILE *out, + const char *block_name, int num_blocks, size_t sizeof_block) +{ + char buf1[128]; + char buf2[128]; + PyOS_snprintf(buf1, sizeof(buf1), + "%d %ss * %zd bytes each", + num_blocks, block_name, sizeof_block); + PyOS_snprintf(buf2, sizeof(buf2), + "%48s ", buf1); + (void)printone(out, buf2, num_blocks * sizeof_block); +} + +#ifdef WITH_PYMALLOC + +/* Print summary info to "out" about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. */ void -_PyObject_DebugMallocStats(void) +_PyObject_DebugMallocStats(FILE *out) { uint i; const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; @@ -1793,7 +1807,7 @@ _PyObject_DebugMallocStats(void) size_t total; char buf[128]; - fprintf(stderr, "Small block threshold = %d, in %u size classes.\n", + fprintf(out, "Small block threshold = %d, in %u size classes.\n", SMALL_REQUEST_THRESHOLD, numclasses); for (i = 0; i < numclasses; ++i) @@ -1847,10 +1861,10 @@ _PyObject_DebugMallocStats(void) } assert(narenas == narenas_currently_allocated); - fputc('\n', stderr); + fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" "----- ---- --------- ------------- ------------\n", - stderr); + out); for (i = 0; i < numclasses; ++i) { size_t p = numpools[i]; @@ -1861,7 +1875,7 @@ _PyObject_DebugMallocStats(void) assert(b == 0 && f == 0); continue; } - fprintf(stderr, "%5u %6u " + fprintf(out, "%5u %6u " "%11" PY_FORMAT_SIZE_T "u " "%15" PY_FORMAT_SIZE_T "u " "%13" PY_FORMAT_SIZE_T "u\n", @@ -1871,35 +1885,36 @@ _PyObject_DebugMallocStats(void) pool_header_bytes += p * POOL_OVERHEAD; quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } - fputc('\n', stderr); - (void)printone("# times object malloc called", serialno); - - (void)printone("# arenas allocated total", ntimes_arena_allocated); - (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas); - (void)printone("# arenas highwater mark", narenas_highwater); - (void)printone("# arenas allocated current", narenas); + fputc('\n', out); +#ifdef PYMALLOC_DEBUG + (void)printone(out, "# times object malloc called", serialno); +#endif + (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", narenas_highwater); + (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), "%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena", narenas, ARENA_SIZE); - (void)printone(buf, narenas * ARENA_SIZE); + (void)printone(out, buf, narenas * ARENA_SIZE); - fputc('\n', stderr); + fputc('\n', out); - total = printone("# bytes in allocated blocks", allocated_bytes); - total += printone("# bytes in available blocks", available_bytes); + total = printone(out, "# bytes in allocated blocks", allocated_bytes); + total += printone(out, "# bytes in available blocks", available_bytes); PyOS_snprintf(buf, sizeof(buf), "%u unused pools * %d bytes", numfreepools, POOL_SIZE); - total += printone(buf, (size_t)numfreepools * POOL_SIZE); + total += printone(out, buf, (size_t)numfreepools * POOL_SIZE); - total += printone("# bytes lost to pool headers", pool_header_bytes); - total += printone("# bytes lost to quantization", quantization); - total += printone("# bytes lost to arena alignment", arena_alignment); - (void)printone("Total", total); + total += printone(out, "# bytes lost to pool headers", pool_header_bytes); + total += printone(out, "# bytes lost to quantization", quantization); + total += printone(out, "# bytes lost to arena alignment", arena_alignment); + (void)printone(out, "Total", total); } -#endif /* PYMALLOC_DEBUG */ +#endif /* #ifdef WITH_PYMALLOC */ #ifdef Py_USING_MEMORY_DEBUGGER /* Make this function last so gcc won't inline it since the definition is diff --git a/Objects/setobject.c b/Objects/setobject.c index b903fbee805..3e91572d32b 100644 --- a/Objects/setobject.c +++ b/Objects/setobject.c @@ -1133,6 +1133,16 @@ PySet_Fini(void) Py_CLEAR(emptyfrozenset); } +/* Print summary info about the state of the optimized allocator */ +void +_PySet_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PySetObject", + numfree, sizeof(PySetObject)); +} + + static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { diff --git a/Objects/tupleobject.c b/Objects/tupleobject.c index d103b9be673..013db69414f 100644 --- a/Objects/tupleobject.c +++ b/Objects/tupleobject.c @@ -45,6 +45,22 @@ show_track(void) } #endif +/* Print summary info about the state of the optimized allocator */ +void +_PyTuple_DebugMallocStats(FILE *out) +{ +#if PyTuple_MAXSAVESIZE > 0 + int i; + char buf[128]; + for (i = 1; i < PyTuple_MAXSAVESIZE; i++) { + PyOS_snprintf(buf, sizeof(buf), + "free %d-sized PyTupleObject", i); + _PyDebugAllocatorStats(out, + buf, + numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, i)); + } +#endif +} PyObject * PyTuple_New(register Py_ssize_t size) diff --git a/Python/pythonrun.c b/Python/pythonrun.c index d9d2fdda812..465aa7dbfea 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -642,7 +642,7 @@ Py_Finalize(void) #endif /* Py_TRACE_REFS */ #ifdef PYMALLOC_DEBUG if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(); + _PyObject_DebugMallocStats(stderr); #endif call_ll_exitfuncs(); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 096e51e9f90..ce5e8258d12 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -997,6 +997,27 @@ a 11-tuple where the entries in the tuple are counts of:\n\ extern "C" { #endif +static PyObject * +sys_debugmallocstats(PyObject *self, PyObject *args) +{ +#ifdef WITH_PYMALLOC + _PyObject_DebugMallocStats(stderr); + fputc('\n', stderr); +#endif + _PyObject_DebugTypeStats(stderr); + + Py_RETURN_NONE; +} +PyDoc_STRVAR(debugmallocstats_doc, +"_debugmallocstats()\n\ +\n\ +Print summary info to stderr about the state of\n\ +pymalloc's structures.\n\ +\n\ +In Py_DEBUG mode, also perform some expensive internal consistency\n\ +checks.\n\ +"); + #ifdef Py_TRACE_REFS /* Defined in objects.c because it uses static globals if that file */ extern PyObject *_Py_GetObjects(PyObject *, PyObject *); @@ -1093,6 +1114,8 @@ static PyMethodDef sys_methods[] = { {"settrace", sys_settrace, METH_O, settrace_doc}, {"gettrace", sys_gettrace, METH_NOARGS, gettrace_doc}, {"call_tracing", sys_call_tracing, METH_VARARGS, call_tracing_doc}, + {"_debugmallocstats", sys_debugmallocstats, METH_VARARGS, + debugmallocstats_doc}, {NULL, NULL} /* sentinel */ };