bpo-37961, tracemalloc: add Traceback.total_nframe (GH-15545)
Add a total_nframe field to the traces collected by the tracemalloc module. This field indicates the original number of frames before it was truncated.
This commit is contained in:
Julien Danjou
Victor Stinner
parent
f3ef06a7cb
commit
8d59eb1b66
|
|
@ -313,6 +313,9 @@ Functions
|
|||
frames. By default, a trace of a memory block only stores the most recent
|
||||
frame: the limit is ``1``. *nframe* must be greater or equal to ``1``.
|
||||
|
||||
You can still read the original number of total frames that composed the
|
||||
traceback by looking at the :attr:`Traceback.total_nframe` attribute.
|
||||
|
||||
Storing more than ``1`` frame is only useful to compute statistics grouped
|
||||
by ``'traceback'`` or to compute cumulative statistics: see the
|
||||
:meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods.
|
||||
|
|
@ -659,6 +662,9 @@ Traceback
|
|||
|
||||
When a snapshot is taken, tracebacks of traces are limited to
|
||||
:func:`get_traceback_limit` frames. See the :func:`take_snapshot` function.
|
||||
The original number of frames of the traceback is stored in the
|
||||
:attr:`Traceback.total_nframe` attribute. That allows to know if a traceback
|
||||
has been truncated by the traceback limit.
|
||||
|
||||
The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback`
|
||||
instance.
|
||||
|
|
@ -666,6 +672,15 @@ Traceback
|
|||
.. versionchanged:: 3.7
|
||||
Frames are now sorted from the oldest to the most recent, instead of most recent to oldest.
|
||||
|
||||
.. attribute:: total_nframe
|
||||
|
||||
Total number of frames that composed the traceback before truncation.
|
||||
This attribute can be set to ``None`` if the information is not
|
||||
available.
|
||||
|
||||
.. versionchanged:: 3.9
|
||||
The :attr:`Traceback.total_nframe` attribute was added.
|
||||
|
||||
.. method:: format(limit=None, most_recent_first=False)
|
||||
|
||||
Format the traceback as a list of lines with newlines. Use the
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ def allocate_bytes(size):
|
|||
bytes_len = (size - EMPTY_STRING_SIZE)
|
||||
frames = get_frames(nframe, 1)
|
||||
data = b'x' * bytes_len
|
||||
return data, tracemalloc.Traceback(frames)
|
||||
return data, tracemalloc.Traceback(frames, min(len(frames), nframe))
|
||||
|
||||
def create_snapshots():
|
||||
traceback_limit = 2
|
||||
|
|
@ -45,27 +45,27 @@ def create_snapshots():
|
|||
# traceback_frames) tuples. traceback_frames is a tuple of (filename,
|
||||
# line_number) tuples.
|
||||
raw_traces = [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
|
||||
(1, 2, (('a.py', 5), ('b.py', 4))),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4)), 3),
|
||||
|
||||
(2, 66, (('b.py', 1),)),
|
||||
(2, 66, (('b.py', 1),), 1),
|
||||
|
||||
(3, 7, (('<unknown>', 0),)),
|
||||
(3, 7, (('<unknown>', 0),), 1),
|
||||
]
|
||||
snapshot = tracemalloc.Snapshot(raw_traces, traceback_limit)
|
||||
|
||||
raw_traces2 = [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
|
||||
(2, 2, (('a.py', 5), ('b.py', 4))),
|
||||
(2, 5000, (('a.py', 5), ('b.py', 4))),
|
||||
(2, 2, (('a.py', 5), ('b.py', 4)), 3),
|
||||
(2, 5000, (('a.py', 5), ('b.py', 4)), 3),
|
||||
|
||||
(4, 400, (('c.py', 578),)),
|
||||
(4, 400, (('c.py', 578),), 1),
|
||||
]
|
||||
snapshot2 = tracemalloc.Snapshot(raw_traces2, traceback_limit)
|
||||
|
||||
|
|
@ -125,7 +125,7 @@ class TestTracemallocEnabled(unittest.TestCase):
|
|||
|
||||
nframe = tracemalloc.get_traceback_limit()
|
||||
frames = get_frames(nframe, -3)
|
||||
obj_traceback = tracemalloc.Traceback(frames)
|
||||
obj_traceback = tracemalloc.Traceback(frames, min(len(frames), nframe))
|
||||
|
||||
traceback = tracemalloc.get_object_traceback(obj)
|
||||
self.assertIsNotNone(traceback)
|
||||
|
|
@ -167,7 +167,7 @@ class TestTracemallocEnabled(unittest.TestCase):
|
|||
trace = self.find_trace(traces, obj_traceback)
|
||||
|
||||
self.assertIsInstance(trace, tuple)
|
||||
domain, size, traceback = trace
|
||||
domain, size, traceback, length = trace
|
||||
self.assertEqual(size, obj_size)
|
||||
self.assertEqual(traceback, obj_traceback._frames)
|
||||
|
||||
|
|
@ -197,8 +197,8 @@ class TestTracemallocEnabled(unittest.TestCase):
|
|||
|
||||
trace1 = self.find_trace(traces, obj1_traceback)
|
||||
trace2 = self.find_trace(traces, obj2_traceback)
|
||||
domain1, size1, traceback1 = trace1
|
||||
domain2, size2, traceback2 = trace2
|
||||
domain1, size1, traceback1, length1 = trace1
|
||||
domain2, size2, traceback2, length2 = trace2
|
||||
self.assertIs(traceback2, traceback1)
|
||||
|
||||
def test_get_traced_memory(self):
|
||||
|
|
@ -259,6 +259,9 @@ class TestTracemallocEnabled(unittest.TestCase):
|
|||
# take a snapshot
|
||||
snapshot = tracemalloc.take_snapshot()
|
||||
|
||||
# This can vary
|
||||
self.assertGreater(snapshot.traces[1].traceback.total_nframe, 10)
|
||||
|
||||
# write on disk
|
||||
snapshot.dump(support.TESTFN)
|
||||
self.addCleanup(support.unlink, support.TESTFN)
|
||||
|
|
@ -321,7 +324,7 @@ class TestSnapshot(unittest.TestCase):
|
|||
maxDiff = 4000
|
||||
|
||||
def test_create_snapshot(self):
|
||||
raw_traces = [(0, 5, (('a.py', 2),))]
|
||||
raw_traces = [(0, 5, (('a.py', 2),), 10)]
|
||||
|
||||
with contextlib.ExitStack() as stack:
|
||||
stack.enter_context(patch.object(tracemalloc, 'is_tracing',
|
||||
|
|
@ -336,6 +339,7 @@ class TestSnapshot(unittest.TestCase):
|
|||
self.assertEqual(len(snapshot.traces), 1)
|
||||
trace = snapshot.traces[0]
|
||||
self.assertEqual(trace.size, 5)
|
||||
self.assertEqual(trace.traceback.total_nframe, 10)
|
||||
self.assertEqual(len(trace.traceback), 1)
|
||||
self.assertEqual(trace.traceback[0].filename, 'a.py')
|
||||
self.assertEqual(trace.traceback[0].lineno, 2)
|
||||
|
|
@ -351,11 +355,11 @@ class TestSnapshot(unittest.TestCase):
|
|||
# exclude b.py
|
||||
snapshot3 = snapshot.filter_traces((filter1,))
|
||||
self.assertEqual(snapshot3.traces._traces, [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4))),
|
||||
(3, 7, (('<unknown>', 0),)),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4)), 3),
|
||||
(3, 7, (('<unknown>', 0),), 1),
|
||||
])
|
||||
|
||||
# filter_traces() must not touch the original snapshot
|
||||
|
|
@ -364,10 +368,10 @@ class TestSnapshot(unittest.TestCase):
|
|||
# only include two lines of a.py
|
||||
snapshot4 = snapshot3.filter_traces((filter2, filter3))
|
||||
self.assertEqual(snapshot4.traces._traces, [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4)), 3),
|
||||
])
|
||||
|
||||
# No filter: just duplicate the snapshot
|
||||
|
|
@ -388,21 +392,21 @@ class TestSnapshot(unittest.TestCase):
|
|||
# exclude a.py of domain 1
|
||||
snapshot3 = snapshot.filter_traces((filter1,))
|
||||
self.assertEqual(snapshot3.traces._traces, [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(2, 66, (('b.py', 1),)),
|
||||
(3, 7, (('<unknown>', 0),)),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(2, 66, (('b.py', 1),), 1),
|
||||
(3, 7, (('<unknown>', 0),), 1),
|
||||
])
|
||||
|
||||
# include domain 1
|
||||
snapshot3 = snapshot.filter_traces((filter1,))
|
||||
self.assertEqual(snapshot3.traces._traces, [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(2, 66, (('b.py', 1),)),
|
||||
(3, 7, (('<unknown>', 0),)),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(2, 66, (('b.py', 1),), 1),
|
||||
(3, 7, (('<unknown>', 0),), 1),
|
||||
])
|
||||
|
||||
def test_filter_traces_domain_filter(self):
|
||||
|
|
@ -413,17 +417,17 @@ class TestSnapshot(unittest.TestCase):
|
|||
# exclude domain 2
|
||||
snapshot3 = snapshot.filter_traces((filter1,))
|
||||
self.assertEqual(snapshot3.traces._traces, [
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4))),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4))),
|
||||
(2, 66, (('b.py', 1),)),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(0, 10, (('a.py', 2), ('b.py', 4)), 3),
|
||||
(1, 2, (('a.py', 5), ('b.py', 4)), 3),
|
||||
(2, 66, (('b.py', 1),), 1),
|
||||
])
|
||||
|
||||
# include domain 2
|
||||
snapshot3 = snapshot.filter_traces((filter2,))
|
||||
self.assertEqual(snapshot3.traces._traces, [
|
||||
(3, 7, (('<unknown>', 0),)),
|
||||
(3, 7, (('<unknown>', 0),), 1),
|
||||
])
|
||||
|
||||
def test_snapshot_group_by_line(self):
|
||||
|
|
|
|||
|
|
@ -182,15 +182,20 @@ class Traceback(Sequence):
|
|||
Sequence of Frame instances sorted from the oldest frame
|
||||
to the most recent frame.
|
||||
"""
|
||||
__slots__ = ("_frames",)
|
||||
__slots__ = ("_frames", '_total_nframe')
|
||||
|
||||
def __init__(self, frames):
|
||||
def __init__(self, frames, total_nframe=None):
|
||||
Sequence.__init__(self)
|
||||
# frames is a tuple of frame tuples: see Frame constructor for the
|
||||
# format of a frame tuple; it is reversed, because _tracemalloc
|
||||
# returns frames sorted from most recent to oldest, but the
|
||||
# Python API expects oldest to most recent
|
||||
self._frames = tuple(reversed(frames))
|
||||
self._total_nframe = total_nframe
|
||||
|
||||
@property
|
||||
def total_nframe(self):
|
||||
return self._total_nframe
|
||||
|
||||
def __len__(self):
|
||||
return len(self._frames)
|
||||
|
|
@ -221,7 +226,12 @@ class Traceback(Sequence):
|
|||
return str(self[0])
|
||||
|
||||
def __repr__(self):
|
||||
return "<Traceback %r>" % (tuple(self),)
|
||||
s = "<Traceback %r" % tuple(self)
|
||||
if self._total_nframe is None:
|
||||
s += ">"
|
||||
else:
|
||||
s += f" total_nframe={self.total_nframe}>"
|
||||
return s
|
||||
|
||||
def format(self, limit=None, most_recent_first=False):
|
||||
lines = []
|
||||
|
|
@ -280,7 +290,7 @@ class Trace:
|
|||
|
||||
@property
|
||||
def traceback(self):
|
||||
return Traceback(self._trace[2])
|
||||
return Traceback(*self._trace[2:])
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, Trace):
|
||||
|
|
@ -378,7 +388,7 @@ class Filter(BaseFilter):
|
|||
return self._match_frame(filename, lineno)
|
||||
|
||||
def _match(self, trace):
|
||||
domain, size, traceback = trace
|
||||
domain, size, traceback, total_nframe = trace
|
||||
res = self._match_traceback(traceback)
|
||||
if self.domain is not None:
|
||||
if self.inclusive:
|
||||
|
|
@ -398,7 +408,7 @@ class DomainFilter(BaseFilter):
|
|||
return self._domain
|
||||
|
||||
def _match(self, trace):
|
||||
domain, size, traceback = trace
|
||||
domain, size, traceback, total_nframe = trace
|
||||
return (domain == self.domain) ^ (not self.inclusive)
|
||||
|
||||
|
||||
|
|
@ -475,7 +485,7 @@ class Snapshot:
|
|||
tracebacks = {}
|
||||
if not cumulative:
|
||||
for trace in self.traces._traces:
|
||||
domain, size, trace_traceback = trace
|
||||
domain, size, trace_traceback, total_nframe = trace
|
||||
try:
|
||||
traceback = tracebacks[trace_traceback]
|
||||
except KeyError:
|
||||
|
|
@ -496,7 +506,7 @@ class Snapshot:
|
|||
else:
|
||||
# cumulative statistics
|
||||
for trace in self.traces._traces:
|
||||
domain, size, trace_traceback = trace
|
||||
domain, size, trace_traceback, total_nframe = trace
|
||||
for frame in trace_traceback:
|
||||
try:
|
||||
traceback = tracebacks[frame]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
Add a ``total_nframe`` field to the traces collected by the tracemalloc module.
|
||||
This field indicates the original number of frames before it was truncated.
|
||||
|
|
@ -78,15 +78,20 @@ __attribute__((packed))
|
|||
|
||||
typedef struct {
|
||||
Py_uhash_t hash;
|
||||
int nframe;
|
||||
/* Number of frames stored */
|
||||
uint16_t nframe;
|
||||
/* Total number of frames the traceback had */
|
||||
uint16_t total_nframe;
|
||||
frame_t frames[1];
|
||||
} traceback_t;
|
||||
|
||||
#define TRACEBACK_SIZE(NFRAME) \
|
||||
(sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1))
|
||||
|
||||
#define MAX_NFRAME \
|
||||
((INT_MAX - (int)sizeof(traceback_t)) / (int)sizeof(frame_t) + 1)
|
||||
/* The maximum number of frames is either:
|
||||
- The maximum number of frames we can store in `traceback_t.nframe`
|
||||
- The maximum memory size_t we can allocate */
|
||||
static const unsigned long MAX_NFRAME = Py_MIN(UINT16_MAX, ((SIZE_MAX - sizeof(traceback_t)) / sizeof(frame_t) + 1));
|
||||
|
||||
|
||||
static PyObject *unknown_filename = NULL;
|
||||
|
|
@ -308,6 +313,9 @@ hashtable_compare_traceback(_Py_hashtable_t *ht, const void *pkey,
|
|||
if (traceback1->nframe != traceback2->nframe)
|
||||
return 0;
|
||||
|
||||
if (traceback1->total_nframe != traceback2->total_nframe)
|
||||
return 0;
|
||||
|
||||
for (i=0; i < traceback1->nframe; i++) {
|
||||
frame1 = &traceback1->frames[i];
|
||||
frame2 = &traceback2->frames[i];
|
||||
|
|
@ -416,6 +424,7 @@ traceback_hash(traceback_t *traceback)
|
|||
/* the cast might truncate len; that doesn't change hash stability */
|
||||
mult += (Py_uhash_t)(82520UL + len + len);
|
||||
}
|
||||
x ^= traceback->total_nframe;
|
||||
x += 97531UL;
|
||||
return x;
|
||||
}
|
||||
|
|
@ -436,11 +445,13 @@ traceback_get_frames(traceback_t *traceback)
|
|||
}
|
||||
|
||||
for (pyframe = tstate->frame; pyframe != NULL; pyframe = pyframe->f_back) {
|
||||
if (traceback->nframe < _Py_tracemalloc_config.max_nframe) {
|
||||
tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]);
|
||||
assert(traceback->frames[traceback->nframe].filename != NULL);
|
||||
traceback->nframe++;
|
||||
if (traceback->nframe == _Py_tracemalloc_config.max_nframe)
|
||||
break;
|
||||
}
|
||||
if (traceback->total_nframe < UINT16_MAX)
|
||||
traceback->total_nframe++;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -456,6 +467,7 @@ traceback_new(void)
|
|||
/* get frames */
|
||||
traceback = tracemalloc_traceback;
|
||||
traceback->nframe = 0;
|
||||
traceback->total_nframe = 0;
|
||||
traceback_get_frames(traceback);
|
||||
if (traceback->nframe == 0)
|
||||
return &tracemalloc_empty_traceback;
|
||||
|
|
@ -1001,6 +1013,7 @@ tracemalloc_init(void)
|
|||
PyUnicode_InternInPlace(&unknown_filename);
|
||||
|
||||
tracemalloc_empty_traceback.nframe = 1;
|
||||
tracemalloc_empty_traceback.total_nframe = 1;
|
||||
/* borrowed reference */
|
||||
tracemalloc_empty_traceback.frames[0].filename = unknown_filename;
|
||||
tracemalloc_empty_traceback.frames[0].lineno = 0;
|
||||
|
|
@ -1046,10 +1059,10 @@ tracemalloc_start(int max_nframe)
|
|||
PyMemAllocatorEx alloc;
|
||||
size_t size;
|
||||
|
||||
if (max_nframe < 1 || max_nframe > MAX_NFRAME) {
|
||||
if (max_nframe < 1 || (unsigned long) max_nframe > MAX_NFRAME) {
|
||||
PyErr_Format(PyExc_ValueError,
|
||||
"the number of frames must be in range [1; %i]",
|
||||
(int)MAX_NFRAME);
|
||||
"the number of frames must be in range [1; %lu]",
|
||||
MAX_NFRAME);
|
||||
return -1;
|
||||
}
|
||||
|
||||
|
|
@ -1062,7 +1075,6 @@ tracemalloc_start(int max_nframe)
|
|||
return 0;
|
||||
}
|
||||
|
||||
assert(1 <= max_nframe && max_nframe <= MAX_NFRAME);
|
||||
_Py_tracemalloc_config.max_nframe = max_nframe;
|
||||
|
||||
/* allocate a buffer to store a new traceback */
|
||||
|
|
@ -1234,7 +1246,7 @@ trace_to_pyobject(unsigned int domain, trace_t *trace,
|
|||
PyObject *trace_obj = NULL;
|
||||
PyObject *obj;
|
||||
|
||||
trace_obj = PyTuple_New(3);
|
||||
trace_obj = PyTuple_New(4);
|
||||
if (trace_obj == NULL)
|
||||
return NULL;
|
||||
|
||||
|
|
@ -1259,6 +1271,13 @@ trace_to_pyobject(unsigned int domain, trace_t *trace,
|
|||
}
|
||||
PyTuple_SET_ITEM(trace_obj, 2, obj);
|
||||
|
||||
obj = PyLong_FromUnsignedLong(trace->traceback->total_nframe);
|
||||
if (obj == NULL) {
|
||||
Py_DECREF(trace_obj);
|
||||
return NULL;
|
||||
}
|
||||
PyTuple_SET_ITEM(trace_obj, 3, obj);
|
||||
|
||||
return trace_obj;
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue