Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

bpo-36389: Fix _PyBytesWriter in release mode (GH-16624) 

Fix _PyBytesWriter API when Python is built in release mode with
assertions.
This commit is contained in:
Victor Stinner 2019-10-07 22:31:42 +02:00 committed by GitHub
parent 6876257eaa
commit 60ec6efd96
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 48 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
Include/internal/pycore_pymem.h
View File

@ -154,6 +154,21 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator(
PyMemAllocatorDomain domain, PyMemAllocatorDomain domain,
PyMemAllocatorEx *old_alloc); PyMemAllocatorEx *old_alloc);
/* Special bytes broadcast into debug memory blocks at appropriate times.
Strings of these are unlikely to be valid addresses, floats, ints or
7-bit ASCII.
- PYMEM_CLEANBYTE: clean (newly allocated) memory
- PYMEM_DEADBYTE dead (newly freed) memory
- PYMEM_FORBIDDENBYTE: untouchable bytes at each end of a block
Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and
0xFD to use the same values than Windows CRT debug malloc() and free().
If modified, _PyMem_IsPtrFreed() should be updated as well. */
#define PYMEM_CLEANBYTE 0xCD
#define PYMEM_DEADBYTE 0xDD
#define PYMEM_FORBIDDENBYTE 0xFD
/* Heuristic checking if a pointer value is newly allocated /* Heuristic checking if a pointer value is newly allocated
(uninitialized), newly freed or NULL (is equal to zero). (uninitialized), newly freed or NULL (is equal to zero).

21
Objects/bytesobject.c
View File

@ -667,9 +667,6 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
Py_ssize_t len = 0; Py_ssize_t len = 0;
char onechar; /* For byte_converter() */ char onechar; /* For byte_converter() */
Py_ssize_t alloc; Py_ssize_t alloc;
#ifdef Py_DEBUG
char *before;
#endif
fmt++; fmt++;
if (*fmt == '%') { if (*fmt == '%') {
@ -981,8 +978,8 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
if (res == NULL) if (res == NULL)
goto error; goto error;
} }
#ifdef Py_DEBUG #ifndef NDEBUG
before = res; char *before = res;
#endif #endif
/* Write the sign if needed */ /* Write the sign if needed */
@ -1047,7 +1044,7 @@ _PyBytes_FormatEx(const char *format, Py_ssize_t format_len,
} }
Py_XDECREF(temp); Py_XDECREF(temp);
#ifdef Py_DEBUG #ifndef NDEBUG
/* check that we computed the exact size for this write */ /* check that we computed the exact size for this write */
assert((res - before) == alloc); assert((res - before) == alloc);
#endif #endif
@ -3168,8 +3165,9 @@ _PyBytesWriter_Init(_PyBytesWriter *writer)
{ {
/* Set all attributes before small_buffer to 0 */ /* Set all attributes before small_buffer to 0 */
memset(writer, 0, offsetof(_PyBytesWriter, small_buffer)); memset(writer, 0, offsetof(_PyBytesWriter, small_buffer));
#ifdef Py_DEBUG #ifndef NDEBUG
memset(writer->small_buffer, 0xCB, sizeof(writer->small_buffer)); memset(writer->small_buffer, PYMEM_CLEANBYTE,
sizeof(writer->small_buffer));
#endif #endif
} }
@ -3297,8 +3295,9 @@ _PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size)
} }
writer->use_small_buffer = 0; writer->use_small_buffer = 0;
#ifdef Py_DEBUG #ifndef NDEBUG
memset(writer->small_buffer, 0xDB, sizeof(writer->small_buffer)); memset(writer->small_buffer, PYMEM_CLEANBYTE,
sizeof(writer->small_buffer));
#endif #endif
} }
writer->allocated = allocated; writer->allocated = allocated;
@ -3350,7 +3349,7 @@ _PyBytesWriter_Alloc(_PyBytesWriter *writer, Py_ssize_t size)
assert(size >= 0); assert(size >= 0);
writer->use_small_buffer = 1; writer->use_small_buffer = 1;
#ifdef Py_DEBUG #ifndef NDEBUG
writer->allocated = sizeof(writer->small_buffer) - 1; writer->allocated = sizeof(writer->small_buffer) - 1;
/* In debug mode, don't use the full small buffer because it is less /* In debug mode, don't use the full small buffer because it is less
efficient than bytes and bytearray objects to detect buffer underflow efficient than bytes and bytearray objects to detect buffer underflow

59
Objects/obmalloc.c
View File

@ -2031,20 +2031,6 @@ _Py_GetAllocatedBlocks(void)
* it wraps a real allocator, adding extra debugging info to the memory blocks. * it wraps a real allocator, adding extra debugging info to the memory blocks.
*/ */
/* Special bytes broadcast into debug memory blocks at appropriate times.
* Strings of these are unlikely to be valid addresses, floats, ints or
* 7-bit ASCII. If modified, _PyMem_IsPtrFreed() should be updated as well.
*
* Byte patterns 0xCB, 0xBB and 0xFB have been replaced with 0xCD, 0xDD and
* 0xFD to use the same values than Windows CRT debug malloc() and free().
*/
#undef CLEANBYTE
#undef DEADBYTE
#undef FORBIDDENBYTE
#define CLEANBYTE 0xCD /* clean (newly allocated) memory */
#define DEADBYTE 0xDD /* dead (newly freed) memory */
#define FORBIDDENBYTE 0xFD /* untouchable bytes at each end of a block */
/* Uncomment this define to add the "serialno" field */ /* Uncomment this define to add the "serialno" field */
/* #define PYMEM_DEBUG_SERIALNO */ /* #define PYMEM_DEBUG_SERIALNO */
@ -2106,14 +2092,14 @@ p[0: S]
p[S] p[S]
API ID. See PEP 445. This is a character, but seems undocumented. API ID. See PEP 445. This is a character, but seems undocumented.
p[S+1: 2*S] p[S+1: 2*S]
Copies of FORBIDDENBYTE. Used to catch under- writes and reads. Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads.
p[2*S: 2*S+n] p[2*S: 2*S+n]
The requested memory, filled with copies of CLEANBYTE. The requested memory, filled with copies of PYMEM_CLEANBYTE.
Used to catch reference to uninitialized memory. Used to catch reference to uninitialized memory.
&p[2*S] is returned. Note that this is 8-byte aligned if pymalloc &p[2*S] is returned. Note that this is 8-byte aligned if pymalloc
handled the request itself. handled the request itself.
p[2*S+n: 2*S+n+S] p[2*S+n: 2*S+n+S]
Copies of FORBIDDENBYTE. Used to catch over- writes and reads. Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads.
p[2*S+n+S: 2*S+n+2*S] p[2*S+n+S: 2*S+n+2*S]
A serial number, incremented by 1 on each call to _PyMem_DebugMalloc A serial number, incremented by 1 on each call to _PyMem_DebugMalloc
and _PyMem_DebugRealloc. and _PyMem_DebugRealloc.
@ -2170,15 +2156,15 @@ _PyMem_DebugRawAlloc(int use_calloc, void *ctx, size_t nbytes)
/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */ /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
write_size_t(p, nbytes); write_size_t(p, nbytes);
p[SST] = (uint8_t)api->api_id; p[SST] = (uint8_t)api->api_id;
memset(p + SST + 1, FORBIDDENBYTE, SST-1); memset(p + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
if (nbytes > 0 && !use_calloc) { if (nbytes > 0 && !use_calloc) {
memset(data, CLEANBYTE, nbytes); memset(data, PYMEM_CLEANBYTE, nbytes);
} }
/* at tail, write pad (SST bytes) and serialno (SST bytes) */ /* at tail, write pad (SST bytes) and serialno (SST bytes) */
tail = data + nbytes; tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST); memset(tail, PYMEM_FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO #ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, serialno); write_size_t(tail + SST, serialno);
#endif #endif
@ -2204,7 +2190,7 @@ _PyMem_DebugRawCalloc(void *ctx, size_t nelem, size_t elsize)
/* The debug free first checks the 2*SST bytes on each end for sanity (in /* The debug free first checks the 2*SST bytes on each end for sanity (in
particular, that the FORBIDDENBYTEs with the api ID are still intact). particular, that the FORBIDDENBYTEs with the api ID are still intact).
Then fills the original bytes with DEADBYTE. Then fills the original bytes with PYMEM_DEADBYTE.
Then calls the underlying free. Then calls the underlying free.
*/ */
static void static void
@ -2222,7 +2208,7 @@ _PyMem_DebugRawFree(void *ctx, void *p)
_PyMem_DebugCheckAddress(api->api_id, p); _PyMem_DebugCheckAddress(api->api_id, p);
nbytes = read_size_t(q); nbytes = read_size_t(q);
nbytes += PYMEM_DEBUG_EXTRA_BYTES; nbytes += PYMEM_DEBUG_EXTRA_BYTES;
memset(q, DEADBYTE, nbytes); memset(q, PYMEM_DEADBYTE, nbytes);
api->alloc.free(api->alloc.ctx, q); api->alloc.free(api->alloc.ctx, q);
} }
@ -2264,14 +2250,14 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
*/ */
if (original_nbytes <= sizeof(save)) { if (original_nbytes <= sizeof(save)) {
memcpy(save, data, original_nbytes); memcpy(save, data, original_nbytes);
memset(data - 2 * SST, DEADBYTE, memset(data - 2 * SST, PYMEM_DEADBYTE,
original_nbytes + PYMEM_DEBUG_EXTRA_BYTES); original_nbytes + PYMEM_DEBUG_EXTRA_BYTES);
} }
else { else {
memcpy(save, data, ERASED_SIZE); memcpy(save, data, ERASED_SIZE);
memset(head, DEADBYTE, ERASED_SIZE + 2 * SST); memset(head, PYMEM_DEADBYTE, ERASED_SIZE + 2 * SST);
memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE); memcpy(&save[ERASED_SIZE], tail - ERASED_SIZE, ERASED_SIZE);
memset(tail - ERASED_SIZE, DEADBYTE, memset(tail - ERASED_SIZE, PYMEM_DEADBYTE,
ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST); ERASED_SIZE + PYMEM_DEBUG_EXTRA_BYTES - 2 * SST);
} }
@ -2293,10 +2279,10 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
write_size_t(head, nbytes); write_size_t(head, nbytes);
head[SST] = (uint8_t)api->api_id; head[SST] = (uint8_t)api->api_id;
memset(head + SST + 1, FORBIDDENBYTE, SST-1); memset(head + SST + 1, PYMEM_FORBIDDENBYTE, SST-1);
tail = data + nbytes; tail = data + nbytes;
memset(tail, FORBIDDENBYTE, SST); memset(tail, PYMEM_FORBIDDENBYTE, SST);
#ifdef PYMEM_DEBUG_SERIALNO #ifdef PYMEM_DEBUG_SERIALNO
write_size_t(tail + SST, block_serialno); write_size_t(tail + SST, block_serialno);
#endif #endif
@ -2320,7 +2306,8 @@ _PyMem_DebugRawRealloc(void *ctx, void *p, size_t nbytes)
if (nbytes > original_nbytes) { if (nbytes > original_nbytes) {
/* growing: mark new extra memory clean */ /* growing: mark new extra memory clean */
memset(data + original_nbytes, CLEANBYTE, nbytes - original_nbytes); memset(data + original_nbytes, PYMEM_CLEANBYTE,
nbytes - original_nbytes);
} }
return data; return data;
@ -2399,7 +2386,7 @@ _PyMem_DebugCheckAddress(char api, const void *p)
* the tail could lead to a segfault then. * the tail could lead to a segfault then.
*/ */
for (i = SST-1; i >= 1; --i) { for (i = SST-1; i >= 1; --i) {
if (*(q-i) != FORBIDDENBYTE) { if (*(q-i) != PYMEM_FORBIDDENBYTE) {
msg = "bad leading pad byte"; msg = "bad leading pad byte";
goto error; goto error;
} }
@ -2408,7 +2395,7 @@ _PyMem_DebugCheckAddress(char api, const void *p)
nbytes = read_size_t(q - 2*SST); nbytes = read_size_t(q - 2*SST);
tail = q + nbytes; tail = q + nbytes;
for (i = 0; i < SST; ++i) { for (i = 0; i < SST; ++i) {
if (tail[i] != FORBIDDENBYTE) { if (tail[i] != PYMEM_FORBIDDENBYTE) {
msg = "bad trailing pad byte"; msg = "bad trailing pad byte";
goto error; goto error;
} }
@ -2448,7 +2435,7 @@ _PyObject_DebugDumpAddress(const void *p)
fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1); fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1);
ok = 1; ok = 1;
for (i = 1; i <= SST-1; ++i) { for (i = 1; i <= SST-1; ++i) {
if (*(q-i) != FORBIDDENBYTE) { if (*(q-i) != PYMEM_FORBIDDENBYTE) {
ok = 0; ok = 0;
break; break;
} }
@ -2457,11 +2444,11 @@ _PyObject_DebugDumpAddress(const void *p)
fputs("FORBIDDENBYTE, as expected.\n", stderr); fputs("FORBIDDENBYTE, as expected.\n", stderr);
else { else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
FORBIDDENBYTE); PYMEM_FORBIDDENBYTE);
for (i = SST-1; i >= 1; --i) { for (i = SST-1; i >= 1; --i) {
const uint8_t byte = *(q-i); const uint8_t byte = *(q-i);
fprintf(stderr, " at p-%d: 0x%02x", i, byte); fprintf(stderr, " at p-%d: 0x%02x", i, byte);
if (byte != FORBIDDENBYTE) if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr); fputs(" *** OUCH", stderr);
fputc('\n', stderr); fputc('\n', stderr);
} }
@ -2476,7 +2463,7 @@ _PyObject_DebugDumpAddress(const void *p)
fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail); fprintf(stderr, " The %d pad bytes at tail=%p are ", SST, (void *)tail);
ok = 1; ok = 1;
for (i = 0; i < SST; ++i) { for (i = 0; i < SST; ++i) {
if (tail[i] != FORBIDDENBYTE) { if (tail[i] != PYMEM_FORBIDDENBYTE) {
ok = 0; ok = 0;
break; break;
} }
@ -2485,12 +2472,12 @@ _PyObject_DebugDumpAddress(const void *p)
fputs("FORBIDDENBYTE, as expected.\n", stderr); fputs("FORBIDDENBYTE, as expected.\n", stderr);
else { else {
fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
FORBIDDENBYTE); PYMEM_FORBIDDENBYTE);
for (i = 0; i < SST; ++i) { for (i = 0; i < SST; ++i) {
const uint8_t byte = tail[i]; const uint8_t byte = tail[i];
fprintf(stderr, " at tail+%d: 0x%02x", fprintf(stderr, " at tail+%d: 0x%02x",
i, byte); i, byte);
if (byte != FORBIDDENBYTE) if (byte != PYMEM_FORBIDDENBYTE)
fputs(" *** OUCH", stderr); fputs(" *** OUCH", stderr);
fputc('\n', stderr); fputc('\n', stderr);
} }