From ae4cfb1bb3a6a597d6daec2620d321279c15eab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kristj=C3=A1n=20Valur=20J=C3=B3nsson?= Date: Mon, 28 Sep 2009 13:45:02 +0000 Subject: [PATCH] http://bugs.python.org/issue6836 Merging revisions 75103,75104 from trunk to py3k --- Include/objimpl.h | 7 +++ Include/pymem.h | 6 +-- Objects/obmalloc.c | 104 +++++++++++++++++++++++++++++++++++++-------- Parser/parsetok.c | 14 ++++-- 4 files changed, 108 insertions(+), 23 deletions(-) diff --git a/Include/objimpl.h b/Include/objimpl.h index 5a273825375..a9ac41cb467 100644 --- a/Include/objimpl.h +++ b/Include/objimpl.h @@ -108,6 +108,13 @@ PyAPI_FUNC(void) _PyObject_DebugFree(void *p); PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p); PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p); PyAPI_FUNC(void) _PyObject_DebugMallocStats(void); +PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes); +PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes); +PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p); +PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p); +PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes); +PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes); +PyAPI_FUNC(void) _PyMem_DebugFree(void *p); #define PyObject_MALLOC _PyObject_DebugMalloc #define PyObject_Malloc _PyObject_DebugMalloc #define PyObject_REALLOC _PyObject_DebugRealloc diff --git a/Include/pymem.h b/Include/pymem.h index 542aceef253..e2dfe0d3fc3 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -59,9 +59,9 @@ PyAPI_FUNC(void) PyMem_Free(void *); /* Macros. */ #ifdef PYMALLOC_DEBUG /* Redirect all memory operations to Python's debugging allocator. */ -#define PyMem_MALLOC PyObject_MALLOC -#define PyMem_REALLOC PyObject_REALLOC -#define PyMem_FREE PyObject_FREE +#define PyMem_MALLOC _PyMem_DebugMalloc +#define PyMem_REALLOC _PyMem_DebugRealloc +#define PyMem_FREE _PyMem_DebugFree #else /* ! PYMALLOC_DEBUG */ diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index d1586c29f88..b2c053f2f4b 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -1241,6 +1241,10 @@ PyObject_Free(void *p) #define DEADBYTE 0xDB /* dead (newly freed) memory */ #define FORBIDDENBYTE 0xFB /* untouchable bytes at each end of a block */ +/* We tag each block with an API ID in order to tag API violations */ +#define _PYMALLOC_MEM_ID 'm' /* the PyMem_Malloc() API */ +#define _PYMALLOC_OBJ_ID 'o' /* The PyObject_Malloc() API */ + static size_t serialno = 0; /* incremented on each debug {m,re}alloc */ /* serialno is always incremented via calling this routine. The point is @@ -1331,8 +1335,49 @@ p[2*S+n+S: 2*S+n+2*S] instant at which this block was passed out. */ +/* debug replacements for the PyMem_* memory API */ +void * +_PyMem_DebugMalloc(size_t nbytes) +{ + return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes); +} +void * +_PyMem_DebugRealloc(void *p, size_t nbytes) +{ + return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes); +} +void +_PyMem_DebugFree(void *p) +{ + _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p); +} + +/* debug replacements for the PyObject_* memory API */ void * _PyObject_DebugMalloc(size_t nbytes) +{ + return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes); +} +void * +_PyObject_DebugRealloc(void *p, size_t nbytes) +{ + return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes); +} +void +_PyObject_DebugFree(void *p) +{ + _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p); +} +void +_PyObject_DebugCheckAddress(void *p) +{ + _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p); +} + + +/* generic debug memory api, with an "id" to identify the API in use */ +void * +_PyObject_DebugMallocApi(char id, size_t nbytes) { uchar *p; /* base address of malloc'ed block */ uchar *tail; /* p + 2*SST + nbytes == pointer to tail pad bytes */ @@ -1348,12 +1393,15 @@ _PyObject_DebugMalloc(size_t nbytes) if (p == NULL) return NULL; + /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */ write_size_t(p, nbytes); - memset(p + SST, FORBIDDENBYTE, SST); + p[SST] = (uchar)id; + memset(p + SST + 1 , FORBIDDENBYTE, SST-1); if (nbytes > 0) memset(p + 2*SST, CLEANBYTE, nbytes); + /* at tail, write pad (SST bytes) and serialno (SST bytes) */ tail = p + 2*SST + nbytes; memset(tail, FORBIDDENBYTE, SST); write_size_t(tail + SST, serialno); @@ -1362,27 +1410,28 @@ _PyObject_DebugMalloc(size_t nbytes) } /* The debug free first checks the 2*SST bytes on each end for sanity (in - particular, that the FORBIDDENBYTEs are still intact). + particular, that the FORBIDDENBYTEs with the api ID are still intact). Then fills the original bytes with DEADBYTE. Then calls the underlying free. */ void -_PyObject_DebugFree(void *p) +_PyObject_DebugFreeApi(char api, void *p) { uchar *q = (uchar *)p - 2*SST; /* address returned from malloc */ size_t nbytes; if (p == NULL) return; - _PyObject_DebugCheckAddress(p); + _PyObject_DebugCheckAddressApi(api, p); nbytes = read_size_t(q); + nbytes += 4*SST; if (nbytes > 0) memset(q, DEADBYTE, nbytes); PyObject_Free(q); } void * -_PyObject_DebugRealloc(void *p, size_t nbytes) +_PyObject_DebugReallocApi(char api, void *p, size_t nbytes) { uchar *q = (uchar *)p; uchar *tail; @@ -1391,9 +1440,9 @@ _PyObject_DebugRealloc(void *p, size_t nbytes) int i; if (p == NULL) - return _PyObject_DebugMalloc(nbytes); + return _PyObject_DebugMallocApi(api, nbytes); - _PyObject_DebugCheckAddress(p); + _PyObject_DebugCheckAddressApi(api, p); bumpserialno(); original_nbytes = read_size_t(q - 2*SST); total = nbytes + 4*SST; @@ -1403,16 +1452,20 @@ _PyObject_DebugRealloc(void *p, size_t nbytes) if (nbytes < original_nbytes) { /* shrinking: mark old extra memory dead */ - memset(q + nbytes, DEADBYTE, original_nbytes - nbytes); + memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST); } - /* Resize and add decorations. */ + /* Resize and add decorations. We may get a new pointer here, in which + * case we didn't get the chance to mark the old memory with DEADBYTE, + * but we live with that. + */ q = (uchar *)PyObject_Realloc(q - 2*SST, total); if (q == NULL) return NULL; write_size_t(q, nbytes); - for (i = 0; i < SST; ++i) + assert(q[SST] == (uchar)api); + for (i = 1; i < SST; ++i) assert(q[SST + i] == FORBIDDENBYTE); q += 2*SST; tail = q + nbytes; @@ -1431,26 +1484,38 @@ _PyObject_DebugRealloc(void *p, size_t nbytes) /* Check the forbidden bytes on both ends of the memory allocated for p. * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress, * and call Py_FatalError to kill the program. + * The API id, is also checked. */ void -_PyObject_DebugCheckAddress(const void *p) +_PyObject_DebugCheckAddressApi(char api, const void *p) { const uchar *q = (const uchar *)p; + char msgbuf[64]; char *msg; size_t nbytes; const uchar *tail; int i; + char id; if (p == NULL) { msg = "didn't expect a NULL pointer"; goto error; } + /* Check the API id */ + id = (char)q[-SST]; + if (id != api) { + msg = msgbuf; + snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api); + msgbuf[sizeof(msgbuf)-1] = 0; + goto error; + } + /* Check the stuff at the start of p first: if there's underwrite * corruption, the number-of-bytes field may be nuts, and checking * the tail could lead to a segfault then. */ - for (i = SST; i >= 1; --i) { + for (i = SST-1; i >= 1; --i) { if (*(q-i) != FORBIDDENBYTE) { msg = "bad leading pad byte"; goto error; @@ -1482,19 +1547,24 @@ _PyObject_DebugDumpAddress(const void *p) size_t nbytes, serial; int i; int ok; + char id; - fprintf(stderr, "Debug memory block at address p=%p:\n", p); - if (p == NULL) + fprintf(stderr, "Debug memory block at address p=%p:", p); + if (p == NULL) { + fprintf(stderr, "\n"); return; + } + id = (char)q[-SST]; + fprintf(stderr, " API '%c'\n", id); nbytes = read_size_t(q - 2*SST); fprintf(stderr, " %" PY_FORMAT_SIZE_T "u bytes originally " "requested\n", nbytes); /* In case this is nuts, check the leading pad bytes first. */ - fprintf(stderr, " The %d pad bytes at p-%d are ", SST, SST); + fprintf(stderr, " The %d pad bytes at p-%d are ", SST-1, SST-1); ok = 1; - for (i = 1; i <= SST; ++i) { + for (i = 1; i <= SST-1; ++i) { if (*(q-i) != FORBIDDENBYTE) { ok = 0; break; @@ -1505,7 +1575,7 @@ _PyObject_DebugDumpAddress(const void *p) else { fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n", FORBIDDENBYTE); - for (i = SST; i >= 1; --i) { + for (i = SST-1; i >= 1; --i) { const uchar byte = *(q-i); fprintf(stderr, " at p-%d: 0x%02x", i, byte); if (byte != FORBIDDENBYTE) diff --git a/Parser/parsetok.c b/Parser/parsetok.c index 14703273c95..90a90a7e73d 100644 --- a/Parser/parsetok.c +++ b/Parser/parsetok.c @@ -240,16 +240,24 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret, } } } else if (tok->encoding != NULL) { + /* 'nodes->n_str' uses PyObject_*, while 'tok->encoding' was + * allocated using PyMem_ + */ node* r = PyNode_New(encoding_decl); - if (!r) { + if (r) + r->n_str = PyObject_MALLOC(strlen(tok->encoding)+1); + if (!r || !r->n_str) { err_ret->error = E_NOMEM; + if (r) + PyObject_FREE(r); n = NULL; goto done; } - r->n_str = tok->encoding; + strcpy(r->n_str, tok->encoding); + PyMem_FREE(tok->encoding); + tok->encoding = NULL; r->n_nchildren = 1; r->n_child = n; - tok->encoding = NULL; n = r; }