Mirror
/
cpython
mirror of https://github.com/python/cpython
4
1
Fork 0
Code Issues Releases Wiki Activity

gh-93649: Split tracemalloc tests from _testcapimodule.c (#99551)

This commit is contained in:
Erlend E. Aasland 2022-12-17 09:53:36 +01:00 committed by GitHub
parent 0264f634f7
commit 2b38a9aa74
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 250 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

169
Lib/test/test_capi/test_mem.py Normal file
View File

@ -0,0 +1,169 @@
import re
import textwrap
import unittest
from test import support
from test.support import import_helper, requires_subprocess
from test.support.script_helper import assert_python_failure, assert_python_ok
# Skip this test if the _testcapi module isn't available.
_testcapi = import_helper.import_module('_testcapi')
@requires_subprocess()
class PyMemDebugTests(unittest.TestCase):
PYTHONMALLOC = 'debug'
# '0x04c06e0' or '04C06E0'
PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+'
def check(self, code):
with support.SuppressCrashReport():
out = assert_python_failure(
'-c', code,
PYTHONMALLOC=self.PYTHONMALLOC,
# FreeBSD: instruct jemalloc to not fill freed() memory
# with junk byte 0x5a, see JEMALLOC(3)
MALLOC_CONF="junk:false",
)
stderr = out.err
return stderr.decode('ascii', 'replace')
def test_buffer_overflow(self):
out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()')
regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
r" 16 bytes originally requested\n"
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
r" at tail\+0: 0x78 \*\*\* OUCH\n"
r" at tail\+1: 0xfd\n"
r" at tail\+2: 0xfd\n"
r" .*\n"
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
r"\n"
r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte")
regex = regex.format(ptr=self.PTR_REGEX)
regex = re.compile(regex, flags=re.DOTALL)
self.assertRegex(out, regex)
def test_api_misuse(self):
out = self.check('import _testcapi; _testcapi.pymem_api_misuse()')
regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
r" 16 bytes originally requested\n"
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
r"\n"
r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n")
regex = regex.format(ptr=self.PTR_REGEX)
self.assertRegex(out, regex)
def check_malloc_without_gil(self, code):
out = self.check(code)
expected = ('Fatal Python error: _PyMem_DebugMalloc: '
'Python memory allocator called without holding the GIL')
self.assertIn(expected, out)
def test_pymem_malloc_without_gil(self):
# Debug hooks must raise an error if PyMem_Malloc() is called
# without holding the GIL
code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
self.check_malloc_without_gil(code)
def test_pyobject_malloc_without_gil(self):
# Debug hooks must raise an error if PyObject_Malloc() is called
# without holding the GIL
code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
self.check_malloc_without_gil(code)
def check_pyobject_is_freed(self, func_name):
code = textwrap.dedent(f'''
import gc, os, sys, _testcapi
# Disable the GC to avoid crash on GC collection
gc.disable()
try:
_testcapi.{func_name}()
# Exit immediately to avoid a crash while deallocating
# the invalid object
os._exit(0)
except _testcapi.error:
os._exit(1)
''')
assert_python_ok(
'-c', code,
PYTHONMALLOC=self.PYTHONMALLOC,
MALLOC_CONF="junk:false",
)
def test_pyobject_null_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_null_is_freed')
def test_pyobject_uninitialized_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
def test_pyobject_forbidden_bytes_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
def test_pyobject_freed_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
def test_set_nomemory(self):
code = """if 1:
import _testcapi
class C(): pass
# The first loop tests both functions and that remove_mem_hooks()
# can be called twice in a row. The second loop checks a call to
# set_nomemory() after a call to remove_mem_hooks(). The third
# loop checks the start and stop arguments of set_nomemory().
for outer_cnt in range(1, 4):
start = 10 * outer_cnt
for j in range(100):
if j == 0:
if outer_cnt != 3:
_testcapi.set_nomemory(start)
else:
_testcapi.set_nomemory(start, start + 1)
try:
C()
except MemoryError as e:
if outer_cnt != 3:
_testcapi.remove_mem_hooks()
print('MemoryError', outer_cnt, j)
_testcapi.remove_mem_hooks()
break
"""
rc, out, err = assert_python_ok('-c', code)
lines = out.splitlines()
for i, line in enumerate(lines, 1):
self.assertIn(b'MemoryError', out)
*_, count = line.split(b' ')
count = int(count)
self.assertLessEqual(count, i*5)
self.assertGreaterEqual(count, i*5-2)
class PyMemMallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'malloc_debug'
@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
class PyMemPymallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'pymalloc_debug'
@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
class PyMemDefaultTests(PyMemDebugTests):
# test default allocator of Python compiled in debug mode
PYTHONMALLOC = ''
if __name__ == "__main__":
unittest.main()

154
Lib/test/test_capi/test_misc.py
View File

@ -323,42 +323,6 @@ class CAPITest(unittest.TestCase):
def test_buildvalue_N(self):
_testcapi.test_buildvalue_N()
def test_set_nomemory(self):
code = """if 1:
import _testcapi
class C(): pass
# The first loop tests both functions and that remove_mem_hooks()
# can be called twice in a row. The second loop checks a call to
# set_nomemory() after a call to remove_mem_hooks(). The third
# loop checks the start and stop arguments of set_nomemory().
for outer_cnt in range(1, 4):
start = 10 * outer_cnt
for j in range(100):
if j == 0:
if outer_cnt != 3:
_testcapi.set_nomemory(start)
else:
_testcapi.set_nomemory(start, start + 1)
try:
C()
except MemoryError as e:
if outer_cnt != 3:
_testcapi.remove_mem_hooks()
print('MemoryError', outer_cnt, j)
_testcapi.remove_mem_hooks()
break
"""
rc, out, err = assert_python_ok('-c', code)
lines = out.splitlines()
for i, line in enumerate(lines, 1):
self.assertIn(b'MemoryError', out)
*_, count = line.split(b' ')
count = int(count)
self.assertLessEqual(count, i*5)
self.assertGreaterEqual(count, i*5-2)
def test_mapping_keys_values_items(self):
class Mapping1(dict):
def keys(self):
@ -1470,124 +1434,6 @@ class Test_testinternalcapi(unittest.TestCase):
if name.startswith('test_'))
@support.requires_subprocess()
class PyMemDebugTests(unittest.TestCase):
PYTHONMALLOC = 'debug'
# '0x04c06e0' or '04C06E0'
PTR_REGEX = r'(?:0x)?[0-9a-fA-F]+'
def check(self, code):
with support.SuppressCrashReport():
out = assert_python_failure(
'-c', code,
PYTHONMALLOC=self.PYTHONMALLOC,
# FreeBSD: instruct jemalloc to not fill freed() memory
# with junk byte 0x5a, see JEMALLOC(3)
MALLOC_CONF="junk:false",
)
stderr = out.err
return stderr.decode('ascii', 'replace')
def test_buffer_overflow(self):
out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()')
regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
r" 16 bytes originally requested\n"
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
r" The [0-9] pad bytes at tail={ptr} are not all FORBIDDENBYTE \(0x[0-9a-f]{{2}}\):\n"
r" at tail\+0: 0x78 \*\*\* OUCH\n"
r" at tail\+1: 0xfd\n"
r" at tail\+2: 0xfd\n"
r" .*\n"
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
r"\n"
r"Fatal Python error: _PyMem_DebugRawFree: bad trailing pad byte")
regex = regex.format(ptr=self.PTR_REGEX)
regex = re.compile(regex, flags=re.DOTALL)
self.assertRegex(out, regex)
def test_api_misuse(self):
out = self.check('import _testcapi; _testcapi.pymem_api_misuse()')
regex = (r"Debug memory block at address p={ptr}: API 'm'\n"
r" 16 bytes originally requested\n"
r" The [0-9] pad bytes at p-[0-9] are FORBIDDENBYTE, as expected.\n"
r" The [0-9] pad bytes at tail={ptr} are FORBIDDENBYTE, as expected.\n"
r"( The block was made by call #[0-9]+ to debug malloc/realloc.\n)?"
r" Data at p: cd cd cd .*\n"
r"\n"
r"Enable tracemalloc to get the memory block allocation traceback\n"
r"\n"
r"Fatal Python error: _PyMem_DebugRawFree: bad ID: Allocated using API 'm', verified using API 'r'\n")
regex = regex.format(ptr=self.PTR_REGEX)
self.assertRegex(out, regex)
def check_malloc_without_gil(self, code):
out = self.check(code)
expected = ('Fatal Python error: _PyMem_DebugMalloc: '
'Python memory allocator called without holding the GIL')
self.assertIn(expected, out)
def test_pymem_malloc_without_gil(self):
# Debug hooks must raise an error if PyMem_Malloc() is called
# without holding the GIL
code = 'import _testcapi; _testcapi.pymem_malloc_without_gil()'
self.check_malloc_without_gil(code)
def test_pyobject_malloc_without_gil(self):
# Debug hooks must raise an error if PyObject_Malloc() is called
# without holding the GIL
code = 'import _testcapi; _testcapi.pyobject_malloc_without_gil()'
self.check_malloc_without_gil(code)
def check_pyobject_is_freed(self, func_name):
code = textwrap.dedent(f'''
import gc, os, sys, _testcapi
# Disable the GC to avoid crash on GC collection
gc.disable()
try:
_testcapi.{func_name}()
# Exit immediately to avoid a crash while deallocating
# the invalid object
os._exit(0)
except _testcapi.error:
os._exit(1)
''')
assert_python_ok(
'-c', code,
PYTHONMALLOC=self.PYTHONMALLOC,
MALLOC_CONF="junk:false",
)
def test_pyobject_null_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_null_is_freed')
def test_pyobject_uninitialized_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_uninitialized_is_freed')
def test_pyobject_forbidden_bytes_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_forbidden_bytes_is_freed')
def test_pyobject_freed_is_freed(self):
self.check_pyobject_is_freed('check_pyobject_freed_is_freed')
class PyMemMallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'malloc_debug'
@unittest.skipUnless(support.with_pymalloc(), 'need pymalloc')
class PyMemPymallocDebugTests(PyMemDebugTests):
PYTHONMALLOC = 'pymalloc_debug'
@unittest.skipUnless(support.Py_DEBUG, 'need Py_DEBUG')
class PyMemDefaultTests(PyMemDebugTests):
# test default allocator of Python compiled in debug mode
PYTHONMALLOC = ''
@unittest.skipIf(_testmultiphase is None, "test requires _testmultiphase module")
class Test_ModuleStateAccess(unittest.TestCase):
"""Test access to module start (PEP 573)"""

81
Modules/_testcapi/mem.c
View File

@ -596,6 +596,82 @@ check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
#endif
}
// Tracemalloc tests
static PyObject *
tracemalloc_track(PyObject *self, PyObject *args)
{
unsigned int domain;
PyObject *ptr_obj;
Py_ssize_t size;
int release_gil = 0;
if (!PyArg_ParseTuple(args, "IOn|i",
&domain, &ptr_obj, &size, &release_gil))
{
return NULL;
}
void *ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred()) {
return NULL;
}
int res;
if (release_gil) {
Py_BEGIN_ALLOW_THREADS
res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
Py_END_ALLOW_THREADS
}
else {
res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
}
if (res < 0) {
PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error");
return NULL;
}
Py_RETURN_NONE;
}
static PyObject *
tracemalloc_untrack(PyObject *self, PyObject *args)
{
unsigned int domain;
PyObject *ptr_obj;
if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) {
return NULL;
}
void *ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred()) {
return NULL;
}
int res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr);
if (res < 0) {
PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error");
return NULL;
}
Py_RETURN_NONE;
}
static PyObject *
tracemalloc_get_traceback(PyObject *self, PyObject *args)
{
unsigned int domain;
PyObject *ptr_obj;
if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj)) {
return NULL;
}
void *ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred()) {
return NULL;
}
return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr);
}
static PyMethodDef test_methods[] = {
{"check_pyobject_forbidden_bytes_is_freed",
check_pyobject_forbidden_bytes_is_freed, METH_NOARGS},
@ -617,6 +693,11 @@ static PyMethodDef test_methods[] = {
{"test_pymem_setrawallocators", test_pymem_setrawallocators, METH_NOARGS},
{"test_pyobject_new", test_pyobject_new, METH_NOARGS},
{"test_pyobject_setallocators", test_pyobject_setallocators, METH_NOARGS},
// Tracemalloc tests
{"tracemalloc_track", tracemalloc_track, METH_VARARGS},
{"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
{"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
{NULL},
};

75
Modules/_testcapimodule.c
View File

@ -2065,78 +2065,6 @@ getitem_with_error(PyObject *self, PyObject *args)
return PyObject_GetItem(map, key);
}
static PyObject *
tracemalloc_track(PyObject *self, PyObject *args)
{
unsigned int domain;
PyObject *ptr_obj;
void *ptr;
Py_ssize_t size;
int release_gil = 0;
int res;
if (!PyArg_ParseTuple(args, "IOn|i", &domain, &ptr_obj, &size, &release_gil))
return NULL;
ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred())
return NULL;
if (release_gil) {
Py_BEGIN_ALLOW_THREADS
res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
Py_END_ALLOW_THREADS
}
else {
res = PyTraceMalloc_Track(domain, (uintptr_t)ptr, size);
}
if (res < 0) {
PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Track error");
return NULL;
}
Py_RETURN_NONE;
}
static PyObject *
tracemalloc_untrack(PyObject *self, PyObject *args)
{
unsigned int domain;
PyObject *ptr_obj;
void *ptr;
int res;
if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
return NULL;
ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred())
return NULL;
res = PyTraceMalloc_Untrack(domain, (uintptr_t)ptr);
if (res < 0) {
PyErr_SetString(PyExc_RuntimeError, "PyTraceMalloc_Untrack error");
return NULL;
}
Py_RETURN_NONE;
}
static PyObject *
tracemalloc_get_traceback(PyObject *self, PyObject *args)
{
unsigned int domain;
PyObject *ptr_obj;
void *ptr;
if (!PyArg_ParseTuple(args, "IO", &domain, &ptr_obj))
return NULL;
ptr = PyLong_AsVoidPtr(ptr_obj);
if (PyErr_Occurred())
return NULL;
return _PyTraceMalloc_GetTraceback(domain, (uintptr_t)ptr);
}
static PyObject *
dict_get_version(PyObject *self, PyObject *args)
{
@ -3301,9 +3229,6 @@ static PyMethodDef TestMethods[] = {
{"return_result_with_error", return_result_with_error, METH_NOARGS},
{"getitem_with_error", getitem_with_error, METH_VARARGS},
{"Py_CompileString", pycompilestring, METH_O},
{"tracemalloc_track", tracemalloc_track, METH_VARARGS},
{"tracemalloc_untrack", tracemalloc_untrack, METH_VARARGS},
{"tracemalloc_get_traceback", tracemalloc_get_traceback, METH_VARARGS},
{"dict_get_version", dict_get_version, METH_VARARGS},
{"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
{"stack_pointer", stack_pointer, METH_NOARGS},