http://bugs.python.org/issue6836

The debug memory api now keeps track of which external API (PyMem_* or PyObject_*) was used to allocate each block and treats any API violation as an error.  Added separate _PyMem_DebugMalloc functions for the Py_Mem API instead of having it use the _PyObject_DebugMalloc functions.

Include/objimpl.h

@@ -108,6 +108,13 @@ PyAPI_FUNC(void) _PyObject_DebugFree(void *p);
 PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p);
 PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p);
 PyAPI_FUNC(void) _PyObject_DebugMallocStats(void);
+PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes);
+PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p);
+PyAPI_FUNC(void) _PyObject_DebugCheckAddressApi(char api, const void *p);
+PyAPI_FUNC(void *) _PyMem_DebugMalloc(size_t nbytes);
+PyAPI_FUNC(void *) _PyMem_DebugRealloc(void *p, size_t nbytes);
+PyAPI_FUNC(void) _PyMem_DebugFree(void *p);
 #define PyObject_MALLOC		_PyObject_DebugMalloc
 #define PyObject_Malloc		_PyObject_DebugMalloc
 #define PyObject_REALLOC	_PyObject_DebugRealloc

Include/pymem.h

@@ -59,9 +59,9 @@ PyAPI_FUNC(void) PyMem_Free(void *);
 /* Macros. */
 #ifdef PYMALLOC_DEBUG
 /* Redirect all memory operations to Python's debugging allocator. */
-#define PyMem_MALLOC		PyObject_MALLOC
+#define PyMem_MALLOC		_PyMem_DebugMalloc
-#define PyMem_REALLOC		PyObject_REALLOC
+#define PyMem_REALLOC		_PyMem_DebugRealloc
-#define PyMem_FREE		PyObject_FREE
+#define PyMem_FREE		_PyMem_DebugFree
 
 #else	/* ! PYMALLOC_DEBUG */
 

Objects/obmalloc.c

@@ -1241,6 +1241,10 @@ PyObject_Free(void *p)
 #define DEADBYTE       0xDB    /* dead (newly freed) memory */
 #define FORBIDDENBYTE  0xFB    /* untouchable bytes at each end of a block */
 
+/* We tag each block with an API ID in order to tag API violations */
+#define _PYMALLOC_MEM_ID 'm'   /* the PyMem_Malloc() API */
+#define _PYMALLOC_OBJ_ID 'o'   /* The PyObject_Malloc() API */
+
 static size_t serialno = 0;	/* incremented on each debug {m,re}alloc */
 
 /* serialno is always incremented via calling this routine.  The point is
@@ -1331,8 +1335,49 @@ p[2*S+n+S: 2*S+n+2*S]
     instant at which this block was passed out.
 */
 
+/* debug replacements for the PyMem_* memory API */
+void *
+_PyMem_DebugMalloc(size_t nbytes)
+{
+	return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID, nbytes);
+}
+void *
+_PyMem_DebugRealloc(void *p, size_t nbytes)
+{
+	return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID, p, nbytes);
+}
+void
+_PyMem_DebugFree(void *p)
+{
+	_PyObject_DebugFreeApi(_PYMALLOC_MEM_ID, p);
+}
+
+/* debug replacements for the PyObject_* memory API */
 void *
 _PyObject_DebugMalloc(size_t nbytes)
+{
+	return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID, nbytes);
+}
+void *
+_PyObject_DebugRealloc(void *p, size_t nbytes)
+{
+	return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID, p, nbytes);
+}
+void
+_PyObject_DebugFree(void *p)
+{
+	_PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID, p);
+}
+void
+_PyObject_DebugCheckAddress(void *p)
+{
+	_PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID, p);
+}
+
+
+/* generic debug memory api, with an "id" to identify the API in use */
+void *
+_PyObject_DebugMallocApi(char id, size_t nbytes)
 {
 	uchar *p;	/* base address of malloc'ed block */
 	uchar *tail;	/* p + 2*SST + nbytes == pointer to tail pad bytes */
@@ -1348,12 +1393,15 @@ _PyObject_DebugMalloc(size_t nbytes)
 	if (p == NULL)
 		return NULL;
 
+	/* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
 	write_size_t(p, nbytes);
-	memset(p + SST, FORBIDDENBYTE, SST);
+	p[SST] = (uchar)id;
+	memset(p + SST + 1 , FORBIDDENBYTE, SST-1);
 
 	if (nbytes > 0)
 		memset(p + 2*SST, CLEANBYTE, nbytes);
 
+	/* at tail, write pad (SST bytes) and serialno (SST bytes) */
 	tail = p + 2*SST + nbytes;
 	memset(tail, FORBIDDENBYTE, SST);
 	write_size_t(tail + SST, serialno);
@@ -1362,27 +1410,28 @@ _PyObject_DebugMalloc(size_t nbytes)
 }
 
 /* The debug free first checks the 2*SST bytes on each end for sanity (in
-   particular, that the FORBIDDENBYTEs are still intact).
+   particular, that the FORBIDDENBYTEs with the api ID are still intact).
    Then fills the original bytes with DEADBYTE.
    Then calls the underlying free.
 */
 void
-_PyObject_DebugFree(void *p)
+_PyObject_DebugFreeApi(char api, void *p)
 {
 	uchar *q = (uchar *)p - 2*SST;  /* address returned from malloc */
 	size_t nbytes;
 
 	if (p == NULL)
 		return;
-	_PyObject_DebugCheckAddress(p);
+	_PyObject_DebugCheckAddressApi(api, p);
 	nbytes = read_size_t(q);
+	nbytes += 4*SST;
 	if (nbytes > 0)
 		memset(q, DEADBYTE, nbytes);
 	PyObject_Free(q);
 }
 
 void *
-_PyObject_DebugRealloc(void *p, size_t nbytes)
+_PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
 {
 	uchar *q = (uchar *)p;
 	uchar *tail;
@@ -1391,9 +1440,9 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 	int i;
 
 	if (p == NULL)
-		return _PyObject_DebugMalloc(nbytes);
+		return _PyObject_DebugMallocApi(api, nbytes);
 
-	_PyObject_DebugCheckAddress(p);
+	_PyObject_DebugCheckAddressApi(api, p);
 	bumpserialno();
 	original_nbytes = read_size_t(q - 2*SST);
 	total = nbytes + 4*SST;
@@ -1403,16 +1452,20 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 
 	if (nbytes < original_nbytes) {
 		/* shrinking:  mark old extra memory dead */
-		memset(q + nbytes, DEADBYTE, original_nbytes - nbytes);
+		memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST);
 	}
 
-	/* Resize and add decorations. */
+	/* Resize and add decorations. We may get a new pointer here, in which
+	 * case we didn't get the chance to mark the old memory with DEADBYTE,
+	 * but we live with that.
+	 */
 	q = (uchar *)PyObject_Realloc(q - 2*SST, total);
 	if (q == NULL)
 		return NULL;
 
 	write_size_t(q, nbytes);
-	for (i = 0; i < SST; ++i)
+	assert(q[SST] == (uchar)api);
+	for (i = 1; i < SST; ++i)
 		assert(q[SST + i] == FORBIDDENBYTE);
 	q += 2*SST;
 	tail = q + nbytes;
@@ -1431,26 +1484,38 @@ _PyObject_DebugRealloc(void *p, size_t nbytes)
 /* Check the forbidden bytes on both ends of the memory allocated for p.
  * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress,
  * and call Py_FatalError to kill the program.
+ * The API id, is also checked.
  */
  void
-_PyObject_DebugCheckAddress(const void *p)
+_PyObject_DebugCheckAddressApi(char api, const void *p)
 {
 	const uchar *q = (const uchar *)p;
+	char msgbuf[64];
 	char *msg;
 	size_t nbytes;
 	const uchar *tail;
 	int i;
+	char id;
 
 	if (p == NULL) {
 		msg = "didn't expect a NULL pointer";
 		goto error;
 	}
 
+	/* Check the API id */
+	id = (char)q[-SST];
+	if (id != api) {
+		msg = msgbuf;
+		snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api);
+		msgbuf[sizeof(msgbuf)-1] = 0;
+		goto error;
+	}
+
 	/* Check the stuff at the start of p first:  if there's underwrite
 	 * corruption, the number-of-bytes field may be nuts, and checking
 	 * the tail could lead to a segfault then.
 	 */
-	for (i = SST; i >= 1; --i) {
+	for (i = SST-1; i >= 1; --i) {
 		if (*(q-i) != FORBIDDENBYTE) {
 			msg = "bad leading pad byte";
 			goto error;
@@ -1482,19 +1547,24 @@ _PyObject_DebugDumpAddress(const void *p)
 	size_t nbytes, serial;
 	int i;
 	int ok;
+	char id;
 
-	fprintf(stderr, "Debug memory block at address p=%p:\n", p);
+	fprintf(stderr, "Debug memory block at address p=%p:", p);
-	if (p == NULL)
+	if (p == NULL) {
+		fprintf(stderr, "\n");
 		return;
+	}
+	id = (char)q[-SST];
+	fprintf(stderr, " API '%c'\n", id);
 
 	nbytes = read_size_t(q - 2*SST);
 	fprintf(stderr, "    %" PY_FORMAT_SIZE_T "u bytes originally "
 	                "requested\n", nbytes);
 
 	/* In case this is nuts, check the leading pad bytes first. */
-	fprintf(stderr, "    The %d pad bytes at p-%d are ", SST, SST);
+	fprintf(stderr, "    The %d pad bytes at p-%d are ", SST-1, SST-1);
 	ok = 1;
-	for (i = 1; i <= SST; ++i) {
+	for (i = 1; i <= SST-1; ++i) {
 		if (*(q-i) != FORBIDDENBYTE) {
 			ok = 0;
 			break;
@@ -1505,7 +1575,7 @@ _PyObject_DebugDumpAddress(const void *p)
 	else {
 		fprintf(stderr, "not all FORBIDDENBYTE (0x%02x):\n",
 			FORBIDDENBYTE);
-		for (i = SST; i >= 1; --i) {
+		for (i = SST-1; i >= 1; --i) {
 			const uchar byte = *(q-i);
 			fprintf(stderr, "        at p-%d: 0x%02x", i, byte);
 			if (byte != FORBIDDENBYTE)