traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-36611: Disable serialno field of debug memory allocators (#12796) 

Omit serialno field from debug hooks on Python memory allocators to
reduce the memory footprint by 5%.

Enable tracemalloc to get the traceback where a memory block has been
allocated when a fatal memory error is logged to decide where to put
a breakpoint.

Compile Python with PYMEM_DEBUG_SERIALNO defined to get back the
field.
This commit is contained in:
Victor Stinner 2019-04-12 21:54:06 +02:00 committed by GitHub
parent 0fc91eef34
commit e8f9acf034
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 61 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Lib/test/test_capi.py
View File

@ -483,7 +483,7 @@ class PyMemDebugTests(unittest.TestCase):
r" at tail\+1: 0xfd\n"
r" at tail\+2: 0xfd\n"
r" .*\n"
r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
@ -499,7 +499,7 @@ class PyMemDebugTests(unittest.TestCase):
r" 16 bytes originally requested\n"
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
r" The block was made by call #[0-9]+ to debug malloc/realloc.\n"
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"

5
Misc/NEWS.d/next/Core and Builtins/2019-04-12-12-32-39.bpo-36611.zbo9WQ.rst Normal file
View File

@ -0,0 +1,5 @@
Debug memory allocators: disable serialno field by default from debug hooks on
Python memory allocators to reduce the memory footprint by 5%. Enable
:mod:`tracemalloc` to get the traceback where a memory block has been allocated
when a fatal memory error is logged to decide where to put a breakpoint.
Compile Python with ``PYMEM_DEBUG_SERIALNO`` defined to get back the field.

67
Objects/obmalloc.c
View File

@ -1926,6 +1926,10 @@ _Py_GetAllocatedBlocks(void)
#define DEADBYTE 0xDD /* dead (newly freed) memory */
#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */
/* Uncomment this define to add the "serialno" field */
/* #define PYMEM_DEBUG_SERIALNO */
#ifdef PYMEM_DEBUG_SERIALNO
static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
/* serialno is always incremented via calling this routine. The point is
@ -1936,9 +1940,16 @@ bumpserialno(void)
{
++serialno;
}
#endif
#define SST SIZEOF_SIZE_T
#ifdef PYMEM_DEBUG_SERIALNO
# define PYMEM_DEBUG_EXTRA_BYTES 4 * SST
#else
# define PYMEM_DEBUG_EXTRA_BYTES 3 * SST
#endif
/* Read sizeof(size_t) bytes at p as a big-endian size_t. */
static size_t
read_size_t(const void *p)
@ -1991,6 +2002,9 @@ p[2*S+n+S: 2*S+n+2*S]
If "bad memory" is detected later, the serial number gives an
excellent way to set a breakpoint on the next run, to capture the
instant at which this block was passed out.
If PYMEM_DEBUG_SERIALNO is not defined (default), the debug malloc only asks
for 3 * S extra bytes, and omits the last serialno field.
*/
static void *
@ -2000,21 +2014,24 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
uint8_t *p; /* base address of malloc'ed pad block */
uint8_t *data; /* p + 2*SST == pointer to data bytes */
uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
size_t total; /* 2 * SST + nbytes + 2 * SST */
size_t total; /* nbytes + PYMEM_DEBUG_EXTRA_BYTES */
if (nbytes > (size_t)PY_SSIZE_T_MAX - 4 * SST) {
if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
/* integer overflow: can't represent total as a Py_ssize_t */
return NULL;
}
total = nbytes + 4 * SST;
total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
/* Layout: [SSSS IFFF CCCC...CCCC FFFF NNNN]
* ^--- p ^--- data ^--- tail
^--- p ^--- data ^--- tail
S: nbytes stored as size_t
I: API identifier (1 byte)
F: Forbidden bytes (size_t - 1 bytes before, size_t bytes after)
C: Clean bytes used later to store actual data
N: Serial number stored as size_t */
N: Serial number stored as size_t
If PYMEM_DEBUG_SERIALNO is not defined (default), the last NNNN field
is omitted. */
if (use_calloc) {
p = (uint8_t *)api->alloc.calloc(api->alloc.ctx, 1, total);
@ -2027,7 +2044,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
}
data = p + 2*SST;
#ifdef PYMEM_DEBUG_SERIALNO
bumpserialno();
#endif
/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes);
@ -2041,7 +2060,9 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
/* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, serialno);
#endif
return data;
}
@ -2081,7 +2102,7 @@ _PyMem_DebugRawFree(void *ctx, void *p)
_PyMem_DebugCheckAddress(api->api_id, p);
nbytes = read_size_t(q);
nbytes += 4 * SST;
nbytes += PYMEM_DEBUG_EXTRA_BYTES;
memset(q, DEADBYTE, nbytes);
api->alloc.free(api->alloc.ctx, q);
}
@ -2101,7 +2122,6 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
uint8_t *tail; /* data + nbytes == pointer to tail pad bytes */
size_t total; /* 2 * SST + nbytes + 2 * SST */
size_t original_nbytes;
size_t block_serialno;
#define ERASED_SIZE 64
uint8_t save[2*ERASED_SIZE]; /* A copy of erased bytes. */
@ -2110,47 +2130,57 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
data = (uint8_t *)p;
head = data - 2*SST;
original_nbytes = read_size_t(head);
if (nbytes > (size_t)PY_SSIZE_T_MAX - 4*SST) {
if (nbytes > (size_t)PY_SSIZE_T_MAX - PYMEM_DEBUG_EXTRA_BYTES) {
/* integer overflow: can't represent total as a Py_ssize_t */
return NULL;
}
total = nbytes + 4*SST;
total = nbytes + PYMEM_DEBUG_EXTRA_BYTES;
tail = data + original_nbytes;
block_serialno = read_size_t(tail + SST);
#ifdef PYMEM_DEBUG_SERIALNO
size_t block_serialno = read_size_t(tail + SST);
#endif
/* Mark the header, the trailer, ERASED_SIZE bytes at the begin and
ERASED_SIZE bytes at the end as dead and save the copy of erased bytes.
*/
if (original_nbytes <= sizeof(save)) {
memcpy(save, data, original_nbytes);
memset(data - 2*SST, DEADBYTE, original_nbytes + 4*SST);
memset(data - 2 * SST, DEADBYTE,
original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
}
else {
memcpy(save, data, ERASED_SIZE);
memset(head, DEADBYTE, ERASED_SIZE + 2 * SST);
memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
memset(tail - ERASED_SIZE, DEADBYTE, ERASED_SIZE + 2*SST);
memset(tail - ERASED_SIZE, DEADBYTE,
ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
}
/* Resize and add decorations. */
r = (uint8_t *)api->alloc.realloc(api->alloc.ctx, head, total);
if (r == NULL) {
/* if realloc() failed: rewrite header and footer which have
just been erased */
nbytes = original_nbytes;
}
else {
head = r;
#ifdef PYMEM_DEBUG_SERIALNO
bumpserialno();
block_serialno = serialno;
#endif
}
data = head + 2*SST;
write_size_t(head, nbytes);
head[SST] = (uint8_t)api->api_id;
memset(head + SST + 1, FORBIDDENBYTE, SST-1);
data = head + 2*SST;
tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, block_serialno);
#endif
/* Restore saved bytes. */
if (original_nbytes <= sizeof(save)) {
@ -2278,7 +2308,7 @@ _PyObject_DebugDumpAddress(const void *p)
{
const uint8_t *q = (const uint8_t *)p;
const uint8_t *tail;
size_t nbytes, serial;
size_t nbytes;
int i;
int ok;
char id;
@ -2347,9 +2377,11 @@ _PyObject_DebugDumpAddress(const void *p)
}
}
serial = read_size_t(tail + SST);
#ifdef PYMEM_DEBUG_SERIALNO
size_t serial = read_size_t(tail + SST);
fprintf(stderr, " The block was made by call #%" PY_FORMAT_SIZE_T
"u to debug malloc/realloc.\n", serial);
#endif
if (nbytes > 0) {
i = 0;
@ -2575,8 +2607,11 @@ _PyObject_DebugMallocStats(FILE *out)
quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size);
}
fputc('\n', out);
if (_PyMem_DebugEnabled())
#ifdef PYMEM_DEBUG_SERIALNO
if (_PyMem_DebugEnabled()) {
(void)printone(out, "# times object malloc called", serialno);
}
#endif
(void)printone(out, "# arenas allocated total", ntimes_arena_allocated);
(void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas);
(void)printone(out, "# arenas highwater mark", narenas_highwater);