bpo-36389: _PyObject_CheckConsistency() available in release mode (GH-16612)

bpo-36389, bpo-38376: The _PyObject_CheckConsistency() function is
now also available in release mode. For example, it can be used to
debug a crash in the visit_decref() function of the GC.

Modify the following functions to also work in release mode:

* _PyDict_CheckConsistency()
* _PyObject_CheckConsistency()
* _PyType_CheckConsistency()
* _PyUnicode_CheckConsistency()

Other changes:

* _PyMem_IsPtrFreed(ptr) now also returns 1 if ptr is NULL
  (equals to 0).
* _PyBytesWriter_CheckConsistency() now returns 1 and is only used
  with assert().
* Reorder _PyObject_Dump() to write safe fields first, and only
  attempt to render repr() at the end.
This commit is contained in:
Victor Stinner 2019-10-07 18:42:01 +02:00 committed by GitHub
parent 321def805a
commit 6876257eaa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 155 additions and 140 deletions

View File

@ -246,6 +246,10 @@ typedef struct {
} data; /* Canonical, smallest-form Unicode buffer */
} PyUnicodeObject;
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
PyObject *op,
int check_content);
/* Fast access macros */
#define PyUnicode_WSTR_LENGTH(op) \
(PyUnicode_IS_COMPACT_ASCII(op) ? \

View File

@ -11,7 +11,6 @@ extern "C" {
#include "pycore_pystate.h" /* _PyRuntime.gc */
PyAPI_FUNC(int) _PyType_CheckConsistency(PyTypeObject *type);
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(PyObject *op, int check_content);
PyAPI_FUNC(int) _PyDict_CheckConsistency(PyObject *mp, int check_content);
/* Tell the GC to track this object.

View File

@ -155,8 +155,9 @@ PyAPI_FUNC(int) _PyMem_SetDefaultAllocator(
PyMemAllocatorEx *old_alloc);
/* Heuristic checking if a pointer value is newly allocated
(uninitialized) or newly freed. The pointer is not dereferenced, only the
pointer value is checked.
(uninitialized), newly freed or NULL (is equal to zero).
The pointer is not dereferenced, only the pointer value is checked.
The heuristic relies on the debug hooks on Python memory allocators which
fills newly allocated memory with CLEANBYTE (0xCD) and newly freed memory
@ -166,11 +167,13 @@ static inline int _PyMem_IsPtrFreed(void *ptr)
{
uintptr_t value = (uintptr_t)ptr;
#if SIZEOF_VOID_P == 8
return (value == (uintptr_t)0xCDCDCDCDCDCDCDCD
return (value == 0
|| value == (uintptr_t)0xCDCDCDCDCDCDCDCD
|| value == (uintptr_t)0xDDDDDDDDDDDDDDDD
|| value == (uintptr_t)0xFDFDFDFDFDFDFDFD);
#elif SIZEOF_VOID_P == 4
return (value == (uintptr_t)0xCDCDCDCD
return (value == 0
|| value == (uintptr_t)0xCDCDCDCD
|| value == (uintptr_t)0xDDDDDDDD
|| value == (uintptr_t)0xFDFDFDFD);
#else

View File

@ -1032,16 +1032,6 @@ PyAPI_FUNC(int) PyUnicode_IsIdentifier(PyObject *s);
/* === Characters Type APIs =============================================== */
#if defined(Py_DEBUG) && !defined(Py_LIMITED_API)
PyAPI_FUNC(int) _PyUnicode_CheckConsistency(
PyObject *op,
int check_content);
#elif !defined(NDEBUG)
/* For asserts that call _PyUnicode_CheckConsistency(), which would
* otherwise be a problem when building with asserts but without Py_DEBUG. */
#define _PyUnicode_CheckConsistency(op, check_content) PyUnicode_Check(op)
#endif
#ifndef Py_LIMITED_API
# define Py_CPYTHON_UNICODEOBJECT_H
# include "cpython/unicodeobject.h"

View File

@ -719,6 +719,9 @@ class PyMemDebugTests(unittest.TestCase):
''')
assert_python_ok('-c', code, PYTHONMALLOC=self.PYTHONMALLOC)
def test_pyobject_null_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_null_is_freed')
def test_pyobject_uninitialized_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')

View File

@ -985,16 +985,19 @@ class GCCallbackTests(unittest.TestCase):
br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.')
self.assertRegex(stderr,
br'refcount is too small')
self.assertRegex(stderr,
br'object : \[1, 2, 3\]')
self.assertRegex(stderr,
br'type : list')
self.assertRegex(stderr,
br'refcount: 1')
# "address : 0x7fb5062efc18"
# "address : 7FB5062EFC18"
address_regex = br'[0-9a-fA-Fx]+'
self.assertRegex(stderr,
br'address : [0-9a-fA-Fx]+')
br'object address : ' + address_regex)
self.assertRegex(stderr,
br'object refcount : 1')
self.assertRegex(stderr,
br'object type : ' + address_regex)
self.assertRegex(stderr,
br'object type name: list')
self.assertRegex(stderr,
br'object repr : \[1, 2, 3\]')
class GCTogglingTests(unittest.TestCase):

View File

@ -0,0 +1,3 @@
The ``_PyObject_CheckConsistency()`` function is now also available in release
mode. For example, it can be used to debug a crash in the ``visit_decref()``
function of the GC.

View File

@ -4611,6 +4611,14 @@ test_pyobject_is_freed(const char *test_name, PyObject *op)
}
static PyObject*
check_pyobject_null_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
{
PyObject *op = NULL;
return test_pyobject_is_freed("check_pyobject_null_is_freed", op);
}
static PyObject*
check_pyobject_uninitialized_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
{
@ -5475,6 +5483,7 @@ static PyMethodDef TestMethods[] = {
{"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
{"pymem_malloc_without_gil", pymem_malloc_without_gil, METH_NOARGS},
{"pymem_getallocatorsname", test_pymem_getallocatorsname, METH_NOARGS},
{"check_pyobject_null_is_freed", check_pyobject_null_is_freed, METH_NOARGS},
{"check_pyobject_uninitialized_is_freed", check_pyobject_uninitialized_is_freed, METH_NOARGS},
{"check_pyobject_forbidden_bytes_is_freed", check_pyobject_forbidden_bytes_is_freed, METH_NOARGS},
{"check_pyobject_freed_is_freed", check_pyobject_freed_is_freed, METH_NOARGS},

View File

@ -375,7 +375,6 @@ update_refs(PyGC_Head *containers)
static int
visit_decref(PyObject *op, void *data)
{
assert(op != NULL);
_PyObject_ASSERT(op, !_PyObject_IsFreed(op));
if (PyObject_IS_GC(op)) {

View File

@ -3206,10 +3206,10 @@ _PyBytesWriter_GetSize(_PyBytesWriter *writer, char *str)
return str - start;
}
Py_LOCAL_INLINE(void)
#ifndef NDEBUG
Py_LOCAL_INLINE(int)
_PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
{
#ifdef Py_DEBUG
char *start, *end;
if (writer->use_small_buffer) {
@ -3239,15 +3239,16 @@ _PyBytesWriter_CheckConsistency(_PyBytesWriter *writer, char *str)
end = start + writer->allocated;
assert(str != NULL);
assert(start <= str && str <= end);
#endif
return 1;
}
#endif
void*
_PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size)
{
Py_ssize_t allocated, pos;
_PyBytesWriter_CheckConsistency(writer, str);
assert(_PyBytesWriter_CheckConsistency(writer, str));
assert(writer->allocated < size);
allocated = size;
@ -3303,7 +3304,7 @@ _PyBytesWriter_Resize(_PyBytesWriter *writer, void *str, Py_ssize_t size)
writer->allocated = allocated;
str = _PyBytesWriter_AsString(writer) + pos;
_PyBytesWriter_CheckConsistency(writer, str);
assert(_PyBytesWriter_CheckConsistency(writer, str));
return str;
error:
@ -3316,7 +3317,7 @@ _PyBytesWriter_Prepare(_PyBytesWriter *writer, void *str, Py_ssize_t size)
{
Py_ssize_t new_min_size;
_PyBytesWriter_CheckConsistency(writer, str);
assert(_PyBytesWriter_CheckConsistency(writer, str));
assert(size >= 0);
if (size == 0) {
@ -3377,7 +3378,7 @@ _PyBytesWriter_Finish(_PyBytesWriter *writer, void *str)
Py_ssize_t size;
PyObject *result;
_PyBytesWriter_CheckConsistency(writer, str);
assert(_PyBytesWriter_CheckConsistency(writer, str));
size = _PyBytesWriter_GetSize(writer, str);
if (size == 0 && !writer->use_bytearray) {

View File

@ -459,23 +459,26 @@ static PyObject *empty_values[1] = { NULL };
int
_PyDict_CheckConsistency(PyObject *op, int check_content)
{
#ifndef NDEBUG
_PyObject_ASSERT(op, PyDict_Check(op));
#define CHECK(expr) \
do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0)
assert(op != NULL);
CHECK(PyDict_Check(op));
PyDictObject *mp = (PyDictObject *)op;
PyDictKeysObject *keys = mp->ma_keys;
int splitted = _PyDict_HasSplitTable(mp);
Py_ssize_t usable = USABLE_FRACTION(keys->dk_size);
_PyObject_ASSERT(op, 0 <= mp->ma_used && mp->ma_used <= usable);
_PyObject_ASSERT(op, IS_POWER_OF_2(keys->dk_size));
_PyObject_ASSERT(op, 0 <= keys->dk_usable && keys->dk_usable <= usable);
_PyObject_ASSERT(op, 0 <= keys->dk_nentries && keys->dk_nentries <= usable);
_PyObject_ASSERT(op, keys->dk_usable + keys->dk_nentries <= usable);
CHECK(0 <= mp->ma_used && mp->ma_used <= usable);
CHECK(IS_POWER_OF_2(keys->dk_size));
CHECK(0 <= keys->dk_usable && keys->dk_usable <= usable);
CHECK(0 <= keys->dk_nentries && keys->dk_nentries <= usable);
CHECK(keys->dk_usable + keys->dk_nentries <= usable);
if (!splitted) {
/* combined table */
_PyObject_ASSERT(op, keys->dk_refcnt == 1);
CHECK(keys->dk_refcnt == 1);
}
if (check_content) {
@ -484,7 +487,7 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
for (i=0; i < keys->dk_size; i++) {
Py_ssize_t ix = dictkeys_get_index(keys, i);
_PyObject_ASSERT(op, DKIX_DUMMY <= ix && ix <= usable);
CHECK(DKIX_DUMMY <= ix && ix <= usable);
}
for (i=0; i < usable; i++) {
@ -494,32 +497,33 @@ _PyDict_CheckConsistency(PyObject *op, int check_content)
if (key != NULL) {
if (PyUnicode_CheckExact(key)) {
Py_hash_t hash = ((PyASCIIObject *)key)->hash;
_PyObject_ASSERT(op, hash != -1);
_PyObject_ASSERT(op, entry->me_hash == hash);
CHECK(hash != -1);
CHECK(entry->me_hash == hash);
}
else {
/* test_dict fails if PyObject_Hash() is called again */
_PyObject_ASSERT(op, entry->me_hash != -1);
CHECK(entry->me_hash != -1);
}
if (!splitted) {
_PyObject_ASSERT(op, entry->me_value != NULL);
CHECK(entry->me_value != NULL);
}
}
if (splitted) {
_PyObject_ASSERT(op, entry->me_value == NULL);
CHECK(entry->me_value == NULL);
}
}
if (splitted) {
/* splitted table */
for (i=0; i < mp->ma_used; i++) {
_PyObject_ASSERT(op, mp->ma_values[i] != NULL);
CHECK(mp->ma_values[i] != NULL);
}
}
}
#endif
return 1;
#undef CHECK
}

View File

@ -25,13 +25,14 @@ _Py_IDENTIFIER(__isabstractmethod__);
int
_PyObject_CheckConsistency(PyObject *op, int check_content)
{
_PyObject_ASSERT(op, op != NULL);
_PyObject_ASSERT(op, !_PyObject_IsFreed(op));
_PyObject_ASSERT(op, Py_REFCNT(op) >= 1);
#define CHECK(expr) \
do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0)
PyTypeObject *type = op->ob_type;
_PyObject_ASSERT(op, type != NULL);
_PyType_CheckConsistency(type);
CHECK(!_PyObject_IsFreed(op));
CHECK(Py_REFCNT(op) >= 1);
CHECK(op->ob_type != NULL);
_PyType_CheckConsistency(op->ob_type);
if (PyUnicode_Check(op)) {
_PyUnicode_CheckConsistency(op, check_content);
@ -40,6 +41,8 @@ _PyObject_CheckConsistency(PyObject *op, int check_content)
_PyDict_CheckConsistency(op, check_content);
}
return 1;
#undef CHECK
}
@ -463,41 +466,41 @@ _PyObject_IsFreed(PyObject *op)
void
_PyObject_Dump(PyObject* op)
{
if (op == NULL) {
fprintf(stderr, "<object at NULL>\n");
fflush(stderr);
return;
}
if (_PyObject_IsFreed(op)) {
/* It seems like the object memory has been freed:
don't access it to prevent a segmentation fault. */
fprintf(stderr, "<object at %p is freed>\n", op);
fflush(stderr);
return;
}
PyGILState_STATE gil;
PyObject *error_type, *error_value, *error_traceback;
fprintf(stderr, "object : ");
fflush(stderr);
gil = PyGILState_Ensure();
PyErr_Fetch(&error_type, &error_value, &error_traceback);
(void)PyObject_Print(op, stderr, 0);
fflush(stderr);
PyErr_Restore(error_type, error_value, error_traceback);
PyGILState_Release(gil);
/* first, write fields which are the least likely to crash */
fprintf(stderr, "object address : %p\n", (void *)op);
/* XXX(twouters) cast refcount to long until %zd is
universally available */
fprintf(stderr, "\n"
"type : %s\n"
"refcount: %ld\n"
"address : %p\n",
Py_TYPE(op)==NULL ? "NULL" : Py_TYPE(op)->tp_name,
(long)op->ob_refcnt,
(void *)op);
fprintf(stderr, "object refcount : %ld\n", (long)op->ob_refcnt);
fflush(stderr);
PyTypeObject *type = Py_TYPE(op);
fprintf(stderr, "object type : %p\n", type);
fprintf(stderr, "object type name: %s\n",
type==NULL ? "NULL" : type->tp_name);
/* the most dangerous part */
fprintf(stderr, "object repr : ");
fflush(stderr);
PyGILState_STATE gil = PyGILState_Ensure();
PyObject *error_type, *error_value, *error_traceback;
PyErr_Fetch(&error_type, &error_value, &error_traceback);
(void)PyObject_Print(op, stderr, 0);
fflush(stderr);
PyErr_Restore(error_type, error_value, error_traceback);
PyGILState_Release(gil);
fprintf(stderr, "\n");
fflush(stderr);
}
@ -2146,6 +2149,7 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg,
fprintf(stderr, "%s: ", function);
}
fflush(stderr);
if (expr) {
fprintf(stderr, "Assertion \"%s\" failed", expr);
}
@ -2153,26 +2157,18 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg,
fprintf(stderr, "Assertion failed");
}
fflush(stderr);
if (msg) {
fprintf(stderr, ": %s", msg);
}
fprintf(stderr, "\n");
fflush(stderr);
if (obj == NULL) {
fprintf(stderr, "<object at NULL>\n");
}
else if (_PyObject_IsFreed(obj)) {
if (_PyObject_IsFreed(obj)) {
/* It seems like the object memory has been freed:
don't access it to prevent a segmentation fault. */
fprintf(stderr, "<object at %p is freed>\n", obj);
}
else if (Py_TYPE(obj) == NULL) {
fprintf(stderr, "<object at %p: ob_type=NULL>\n", obj);
}
else if (_PyObject_IsFreed((PyObject *)Py_TYPE(obj))) {
fprintf(stderr, "<object at %p: type at %p is freed>\n",
obj, (void *)Py_TYPE(obj));
fflush(stderr);
}
else {
/* Display the traceback where the object has been allocated.
@ -2192,7 +2188,6 @@ _PyObject_AssertFailed(PyObject *obj, const char *expr, const char *msg,
try to provide any extra info we can: */
_PyObject_Dump(obj);
}
fflush(stderr);
Py_FatalError("_PyObject_AssertFailed");
}

View File

@ -137,22 +137,24 @@ skip_signature(const char *doc)
int
_PyType_CheckConsistency(PyTypeObject *type)
{
#define ASSERT(expr) _PyObject_ASSERT((PyObject *)type, (expr))
#define CHECK(expr) \
do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG((PyObject *)type, Py_STRINGIFY(expr)); } } while (0)
CHECK(!_PyObject_IsFreed((PyObject *)type));
if (!(type->tp_flags & Py_TPFLAGS_READY)) {
/* don't check types before PyType_Ready() */
/* don't check static types before PyType_Ready() */
return 1;
}
ASSERT(!_PyObject_IsFreed((PyObject *)type));
ASSERT(Py_REFCNT(type) >= 1);
ASSERT(PyType_Check(type));
CHECK(Py_REFCNT(type) >= 1);
CHECK(PyType_Check(type));
ASSERT(!(type->tp_flags & Py_TPFLAGS_READYING));
ASSERT(type->tp_dict != NULL);
CHECK(!(type->tp_flags & Py_TPFLAGS_READYING));
CHECK(type->tp_dict != NULL);
return 1;
#undef ASSERT
#undef CHECK
}
static const char *

View File

@ -487,65 +487,63 @@ PyUnicode_GetMax(void)
int
_PyUnicode_CheckConsistency(PyObject *op, int check_content)
{
#define CHECK(expr) \
do { if (!(expr)) { _PyObject_ASSERT_FAILED_MSG(op, Py_STRINGIFY(expr)); } } while (0)
PyASCIIObject *ascii;
unsigned int kind;
_PyObject_ASSERT(op, PyUnicode_Check(op));
assert(op != NULL);
CHECK(PyUnicode_Check(op));
ascii = (PyASCIIObject *)op;
kind = ascii->state.kind;
if (ascii->state.ascii == 1 && ascii->state.compact == 1) {
_PyObject_ASSERT(op, kind == PyUnicode_1BYTE_KIND);
_PyObject_ASSERT(op, ascii->state.ready == 1);
CHECK(kind == PyUnicode_1BYTE_KIND);
CHECK(ascii->state.ready == 1);
}
else {
PyCompactUnicodeObject *compact = (PyCompactUnicodeObject *)op;
#ifndef NDEBUG
void *data;
#endif
if (ascii->state.compact == 1) {
#ifndef NDEBUG
data = compact + 1;
#endif
_PyObject_ASSERT(op, kind == PyUnicode_1BYTE_KIND
CHECK(kind == PyUnicode_1BYTE_KIND
|| kind == PyUnicode_2BYTE_KIND
|| kind == PyUnicode_4BYTE_KIND);
_PyObject_ASSERT(op, ascii->state.ascii == 0);
_PyObject_ASSERT(op, ascii->state.ready == 1);
_PyObject_ASSERT(op, compact->utf8 != data);
CHECK(ascii->state.ascii == 0);
CHECK(ascii->state.ready == 1);
CHECK(compact->utf8 != data);
}
else {
#ifndef NDEBUG
PyUnicodeObject *unicode = (PyUnicodeObject *)op;
data = unicode->data.any;
#endif
if (kind == PyUnicode_WCHAR_KIND) {
_PyObject_ASSERT(op, ascii->length == 0);
_PyObject_ASSERT(op, ascii->hash == -1);
_PyObject_ASSERT(op, ascii->state.compact == 0);
_PyObject_ASSERT(op, ascii->state.ascii == 0);
_PyObject_ASSERT(op, ascii->state.ready == 0);
_PyObject_ASSERT(op, ascii->state.interned == SSTATE_NOT_INTERNED);
_PyObject_ASSERT(op, ascii->wstr != NULL);
_PyObject_ASSERT(op, data == NULL);
_PyObject_ASSERT(op, compact->utf8 == NULL);
CHECK(ascii->length == 0);
CHECK(ascii->hash == -1);
CHECK(ascii->state.compact == 0);
CHECK(ascii->state.ascii == 0);
CHECK(ascii->state.ready == 0);
CHECK(ascii->state.interned == SSTATE_NOT_INTERNED);
CHECK(ascii->wstr != NULL);
CHECK(data == NULL);
CHECK(compact->utf8 == NULL);
}
else {
_PyObject_ASSERT(op, kind == PyUnicode_1BYTE_KIND
CHECK(kind == PyUnicode_1BYTE_KIND
|| kind == PyUnicode_2BYTE_KIND
|| kind == PyUnicode_4BYTE_KIND);
_PyObject_ASSERT(op, ascii->state.compact == 0);
_PyObject_ASSERT(op, ascii->state.ready == 1);
_PyObject_ASSERT(op, data != NULL);
CHECK(ascii->state.compact == 0);
CHECK(ascii->state.ready == 1);
CHECK(data != NULL);
if (ascii->state.ascii) {
_PyObject_ASSERT(op, compact->utf8 == data);
_PyObject_ASSERT(op, compact->utf8_length == ascii->length);
CHECK(compact->utf8 == data);
CHECK(compact->utf8_length == ascii->length);
}
else
_PyObject_ASSERT(op, compact->utf8 != data);
CHECK(compact->utf8 != data);
}
}
if (kind != PyUnicode_WCHAR_KIND) {
@ -557,16 +555,16 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
#endif
)
{
_PyObject_ASSERT(op, ascii->wstr == data);
_PyObject_ASSERT(op, compact->wstr_length == ascii->length);
CHECK(ascii->wstr == data);
CHECK(compact->wstr_length == ascii->length);
} else
_PyObject_ASSERT(op, ascii->wstr != data);
CHECK(ascii->wstr != data);
}
if (compact->utf8 == NULL)
_PyObject_ASSERT(op, compact->utf8_length == 0);
CHECK(compact->utf8_length == 0);
if (ascii->wstr == NULL)
_PyObject_ASSERT(op, compact->wstr_length == 0);
CHECK(compact->wstr_length == 0);
}
/* check that the best kind is used: O(n) operation */
@ -585,23 +583,25 @@ _PyUnicode_CheckConsistency(PyObject *op, int check_content)
}
if (kind == PyUnicode_1BYTE_KIND) {
if (ascii->state.ascii == 0) {
_PyObject_ASSERT(op, maxchar >= 128);
_PyObject_ASSERT(op, maxchar <= 255);
CHECK(maxchar >= 128);
CHECK(maxchar <= 255);
}
else
_PyObject_ASSERT(op, maxchar < 128);
CHECK(maxchar < 128);
}
else if (kind == PyUnicode_2BYTE_KIND) {
_PyObject_ASSERT(op, maxchar >= 0x100);
_PyObject_ASSERT(op, maxchar <= 0xFFFF);
CHECK(maxchar >= 0x100);
CHECK(maxchar <= 0xFFFF);
}
else {
_PyObject_ASSERT(op, maxchar >= 0x10000);
_PyObject_ASSERT(op, maxchar <= MAX_UNICODE);
CHECK(maxchar >= 0x10000);
CHECK(maxchar <= MAX_UNICODE);
}
_PyObject_ASSERT(op, PyUnicode_READ(kind, data, ascii->length) == 0);
CHECK(PyUnicode_READ(kind, data, ascii->length) == 0);
}
return 1;
#undef CHECK
}