From e8f9acf03484c6c3f163f04a76321419369c28aa Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 12 Apr 2019 21:54:06 +0200 Subject: [PATCH] bpo-36611: Disable serialno field of debug memory allocators (#12796) Omit serialno field from debug hooks on Python memory allocators to reduce the memory footprint by 5%. Enable tracemalloc to get the traceback where a memory block has been allocated when a fatal memory error is logged to decide where to put a breakpoint. Compile Python with PYMEM_DEBUG_SERIALNO defined to get back the field. --- Lib/test/test_capi.py | 4 +- .../2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst | 5 ++ Objects/obmalloc.c | 73 ++++++++++++++----- 3 files changed, 61 insertions(+), 21 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index 33c98ac28bc..31dab6a423e 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -483,7 +483,7 @@ class PyMemDebugTests(unittest.TestCase): r" at tail\+1: 0xfd\n" r" at tail\+2: 0xfd\n" r" .*\n" - r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" r" Data at p: cd cd cd .*\n" r"\n" r"Enable tracemalloc to get the memory block allocation traceback\n" @@ -499,7 +499,7 @@ class PyMemDebugTests(unittest.TestCase): r" 16 bytes originally requested\n" r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n" r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n" - r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?" r" Data at p: cd cd cd .*\n" r"\n" r"Enable tracemalloc to get the memory block allocation traceback\n" diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst b/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst new file mode 100644 index 00000000000..f55a9efc5d3 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst @@ -0,0 +1,5 @@ +Debug memory allocators: disable serialno field by default from debug hooks on +Python memory allocators to reduce the memory footprint by 5%. Enable +:mod:`tracemalloc` to get the traceback where a memory block has been allocated +when a fatal memory error is logged to decide where to put a breakpoint. +Compile Python with ``PYMEM_DEBUG_SERIALNO`` defined to get back the field. diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index be43c7a1c2b..3ee143549d2 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1926,6 +1926,10 @@ _Py_GetAllocatedBlocks(void) #define DEADBYTE 0xDD /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */ +/* Uncomment this define to add the "serialno" field */ +/* #define PYMEM_DEBUG_SERIALNO */ + +#ifdef PYMEM_DEBUG_SERIALNO static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ /* serialno is always incremented via calling this routine. The point is @@ -1936,9 +1940,16 @@ bumpserialno(void) { ++serialno; } +#endif #define SST SIZEOF_SIZE_T +#ifdef PYMEM_DEBUG_SERIALNO +# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST +#else +# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST +#endif + /* Read sizeof(size_t) bytes at p as a big-endian size_t. */ static size_t read_size_t(const void *p) @@ -1967,7 +1978,7 @@ write_size_t(void *p, size_t n) } } -/* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and +/* Let S = sizeof(size_t). The debug malloc asks for 4 * S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: p[0: S] @@ -1991,6 +2002,9 @@ p[2*S+n+S: 2*S+n+2*S] If "bad memory" is detected later, the serial number gives an excellent way to set a breakpoint on the next run, to capture the instant at which this block was passed out. + +If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks +for 3 * S extra bytes, and omits the last serialno field. */ static void * @@ -2000,21 +2014,24 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) uint8_t *p; /* base address of malloc'ed pad block */ uint8_t *data; /* p + 2*SST == pointer to data bytes */ uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ - size_t total; /* 2 * SST + nbytes + 2 * SST */ + size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */ - if (nbytes > (size_t)PY_SSIZE_T_MAX - 4 * SST) { + if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { /* integer overflow: can't represent total as a Py_ssize_t */ return NULL; } - total = nbytes + 4 * SST; + total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; /* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN] - * ^--- p ^--- data ^--- tail + ^--- p ^--- data ^--- tail S: nbytes stored as size_t I: API identifier (1 byte) F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after) C: Clean bytes used later to store actual data - N: Serial number stored as size_t */ + N: Serial number stored as size_t + + If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field + is omitted. */ if (use_calloc) { p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total); @@ -2027,7 +2044,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) } data = p + 2*SST; +#ifdef PYMEM_DEBUG_SERIALNO bumpserialno(); +#endif /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */ write_size_t(p, nbytes); @@ -2041,7 +2060,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes) /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); +#ifdef PYMEM_DEBUG_SERIALNO write_size_t(tail + SST, serialno); +#endif return data; } @@ -2081,7 +2102,7 @@ _PyMem_DebugRawFree(void *ctx, void *p) _PyMem_DebugCheckAddress(api->api_id, p); nbytes = read_size_t(q); - nbytes += 4 * SST; + nbytes += PYMEM_DEBUG_EXTRA_BYTES; memset(q, DEADBYTE, nbytes); api->alloc.free(api->alloc.ctx, q); } @@ -2101,7 +2122,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */ size_t total; /* 2 * SST + nbytes + 2 * SST */ size_t original_nbytes; - size_t block_serialno; #define ERASED_SIZE 64 uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */ @@ -2110,47 +2130,57 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) data = (uint8_t *)p; head = data - 2*SST; original_nbytes = read_size_t(head); - if (nbytes > (size_t)PY_SSIZE_T_MAX - 4*SST) { + if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) { /* integer overflow: can't represent total as a Py_ssize_t */ return NULL; } - total = nbytes + 4*SST; + total = nbytes + PYMEM_DEBUG_EXTRA_BYTES; tail = data + original_nbytes; - block_serialno = read_size_t(tail + SST); +#ifdef PYMEM_DEBUG_SERIALNO + size_t block_serialno = read_size_t(tail + SST); +#endif /* Mark the header, the trailer, ERASED_SIZE bytes at the begin and ERASED_SIZE bytes at the end as dead and save the copy of erased bytes. */ if (original_nbytes <= sizeof(save)) { memcpy(save, data, original_nbytes); - memset(data - 2*SST, DEADBYTE, original_nbytes + 4*SST); + memset(data - 2 * SST, DEADBYTE, + original_nbytes + PYMEM_DEBUG_EXTRA_BYTES); } else { memcpy(save, data, ERASED_SIZE); - memset(head, DEADBYTE, ERASED_SIZE + 2*SST); + memset(head, DEADBYTE, ERASED_SIZE + 2 * SST); memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE); - memset(tail - ERASED_SIZE, DEADBYTE, ERASED_SIZE + 2*SST); + memset(tail - ERASED_SIZE, DEADBYTE, + ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); } /* Resize and add decorations. */ r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total); if (r == NULL) { + /* if realloc() failed: rewrite header and footer which have + just been erased */ nbytes = original_nbytes; } else { head = r; +#ifdef PYMEM_DEBUG_SERIALNO bumpserialno(); block_serialno = serialno; +#endif } + data = head + 2*SST; write_size_t(head, nbytes); head[SST] = (uint8_t)api->api_id; memset(head + SST + 1, FORBIDDENBYTE, SST-1); - data = head + 2*SST; tail = data + nbytes; memset(tail, FORBIDDENBYTE, SST); +#ifdef PYMEM_DEBUG_SERIALNO write_size_t(tail + SST, block_serialno); +#endif /* Restore saved bytes. */ if (original_nbytes <= sizeof(save)) { @@ -2170,7 +2200,7 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes) } if (nbytes > original_nbytes) { - /* growing: mark new extra memory clean */ + /* growing: mark new extra memory clean */ memset(data + original_nbytes, CLEANBYTE, nbytes - original_nbytes); } @@ -2278,7 +2308,7 @@ _PyObject_DebugDumpAddress(const void *p) { const uint8_t *q = (const uint8_t *)p; const uint8_t *tail; - size_t nbytes, serial; + size_t nbytes; int i; int ok; char id; @@ -2347,9 +2377,11 @@ _PyObject_DebugDumpAddress(const void *p) } } - serial = read_size_t(tail + SST); +#ifdef PYMEM_DEBUG_SERIALNO + size_t serial = read_size_t(tail + SST); fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T "u to debug malloc/realloc.\n", serial); +#endif if (nbytes > 0) { i = 0; @@ -2575,8 +2607,11 @@ _PyObject_DebugMallocStats(FILE *out) quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } fputc('\n', out); - if (_PyMem_DebugEnabled()) +#ifdef PYMEM_DEBUG_SERIALNO + if (_PyMem_DebugEnabled()) { (void)printone(out, "# times object malloc called", serialno); + } +#endif (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas highwater mark", narenas_highwater);