traverseda
/
cpython
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

bpo-42246: Partial implementation of PEP 626. (GH-23113) 

* Implement new line number table format, as defined in PEP 626.
This commit is contained in:
Mark Shannon 2020-11-12 09:43:29 +00:00 committed by GitHub
parent cda99b4022
commit 877df851c3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 5364 additions and 5000 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
Include/cpython/code.h
View File

@ -38,7 +38,7 @@ struct PyCodeObject {
Py_ssize_t *co_cell2arg; /* Maps cell vars which are arguments. */
PyObject *co_filename; /* unicode (where it was loaded from) */
PyObject *co_name; /* unicode (name, for reference) */
PyObject *co_lnotab; /* string (encoding addr<->lineno mapping) See
PyObject *co_linetable; /* string (encoding addr<->lineno mapping) See
Objects/lnotab_notes.txt for details. */
void *co_zombieframe; /* for optimization only (see frameobject.c) */
PyObject *co_weakreflist; /* to support weakrefs to code objects */
@ -135,16 +135,18 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno);
PyAPI_FUNC(int) PyCode_Addr2Line(PyCodeObject *, int);
/* for internal use only */
typedef struct _addr_pair {
int ap_lower;
int ap_upper;
} PyAddrPair;
typedef struct _line_offsets {
int ar_start;
int ar_end;
int ar_line;
int ar_computed_line;
char *lo_next;
} PyCodeAddressRange;
/* Update *bounds to describe the first and one-past-the-last instructions in the
same line as lasti. Return the number of that line.
*/
PyAPI_FUNC(int) _PyCode_CheckLineNumber(PyCodeObject* co,
int lasti, PyAddrPair *bounds);
PyAPI_FUNC(int) _PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds);
/* Create a comparable key used to compare constants taking in account the
* object type. It is used to make sure types are not coerced (e.g., float and
@ -163,3 +165,15 @@ PyAPI_FUNC(int) _PyCode_GetExtra(PyObject *code, Py_ssize_t index,
void **extra);
PyAPI_FUNC(int) _PyCode_SetExtra(PyObject *code, Py_ssize_t index,
void *extra);
/** API for initializing the line number table. */
int _PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds);
/** Out of process API for initializing the line number table. */
void PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range);
/** API for traversing the line number table. */
int PyLineTable_NextAddressRange(PyCodeAddressRange *range);
int PyLineTable_PreviousAddressRange(PyCodeAddressRange *range);

31
Lib/dis.py
View File

@ -449,32 +449,15 @@ def findlabels(code):
def findlinestarts(code):
"""Find the offsets in a byte code which are start of lines in the source.
Generate pairs (offset, lineno) as described in Python/compile.c.
Generate pairs (offset, lineno)
"""
byte_increments = code.co_lnotab[0::2]
line_increments = code.co_lnotab[1::2]
bytecode_len = len(code.co_code)
lastline = None
for start, end, line in code.co_lines():
if line is not None and line != lastline:
lastline = line
yield start, line
return
lastlineno = None
lineno = code.co_firstlineno
addr = 0
for byte_incr, line_incr in zip(byte_increments, line_increments):
if byte_incr:
if lineno != lastlineno:
yield (addr, lineno)
lastlineno = lineno
addr += byte_incr
if addr >= bytecode_len:
# The rest of the lnotab byte offsets are past the end of
# the bytecode, so the lines were optimized away.
return
if line_incr >= 0x80:
# line_increments is an array of 8-bit signed integers
line_incr -= 0x100
lineno += line_incr
if lineno != lastlineno:
yield (addr, lineno)
class Bytecode:
"""The bytecode operations of a piece of code

2
Lib/test/test_code.py
View File

@ -258,7 +258,7 @@ class CodeTest(unittest.TestCase):
("co_cellvars", ("cellvar",)),
("co_filename", "newfilename"),
("co_name", "newname"),
("co_lnotab", code2.co_lnotab),
("co_linetable", code2.co_linetable),
):
with self.subTest(attr=attr, value=value):
new_code = code.replace(**{attr: value})

4
Lib/test/test_compile.py
View File

@ -155,8 +155,8 @@ if 1:
def test_leading_newlines(self):
s256 = "".join(["\n"] * 256 + ["spam"])
co = compile(s256, 'fn', 'exec')
self.assertEqual(co.co_firstlineno, 257)
self.assertEqual(co.co_lnotab, bytes())
self.assertEqual(co.co_firstlineno, 1)
self.assertEqual(list(co.co_lines()), [(0, 4, 257), (4, 8, None)])
def test_literals_with_leading_zeroes(self):
for arg in ["077787", "0xj", "0x.", "0e", "090000000000000",

2
Lib/test/test_opcodes.py
View File

@ -27,7 +27,7 @@ class OpcodeTest(unittest.TestCase):
with open(ann_module.__file__) as f:
txt = f.read()
co = compile(txt, ann_module.__file__, 'exec')
self.assertEqual(co.co_firstlineno, 3)
self.assertEqual(co.co_firstlineno, 1)
except OSError:
pass

3
Lib/test/test_pdb.py
View File

@ -1645,9 +1645,10 @@ def bœr():
'debug doesnotexist',
'c',
])
stdout, _ = self.run_pdb_script('', commands + '\n')
stdout, _ = self.run_pdb_script('pass', commands + '\n')
self.assertEqual(stdout.splitlines()[1:], [
'-> pass',
'(Pdb) *** SyntaxError: unexpected EOF while parsing',
'(Pdb) ENTERING RECURSIVE DEBUGGER',

18
Lib/test/test_sys_settrace.py
View File

@ -220,8 +220,7 @@ ireturn_example.events = [(0, 'call'),
(2, 'line'),
(3, 'line'),
(4, 'line'),
(6, 'line'),
(6, 'return')]
(4, 'return')]
# Tight loop with while(1) example (SF #765624)
def tightloop_example():
@ -602,6 +601,17 @@ class TraceTestCase(unittest.TestCase):
self.compare_events(doit_async.__code__.co_firstlineno,
tracer.events, events)
def test_21_repeated_pass(self):
def func():
pass
pass
self.run_and_compare(func,
[(0, 'call'),
(1, 'line'),
(2, 'line'),
(2, 'return')])
def test_loop_in_try_except(self):
# https://bugs.python.org/issue41670
@ -766,7 +776,7 @@ class JumpTracer:
if (self.firstLine is None and frame.f_code == self.code and
event == 'line'):
self.firstLine = frame.f_lineno - 1
if (event == self.event and self.firstLine and
if (event == self.event and self.firstLine is not None and
frame.f_lineno == self.firstLine + self.jumpFrom):
f = frame
while f is not None and f.f_code != self.code:
@ -1540,7 +1550,7 @@ output.append(4)
""", "<fake module>", "exec")
class fake_function:
__code__ = code
tracer = JumpTracer(fake_function, 2, 0)
tracer = JumpTracer(fake_function, 4, 1)
sys.settrace(tracer.trace)
namespace = {"output": []}
exec(code, namespace)

2
Misc/NEWS.d/next/Core and Builtins/2020-11-02-15-48-17.bpo-42246.3CNQEX.rst Normal file
View File

@ -0,0 +1,2 @@
Improved accuracy of line tracing events and f_lineno attribute of Frame
objects. See PEP 626 for details.

26
Objects/clinic/codeobject.c.h generated
View File

@ -5,7 +5,7 @@ preserve
PyDoc_STRVAR(code_new__doc__,
"code(argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize,\n"
" flags, codestring, constants, names, varnames, filename, name,\n"
" firstlineno, lnotab, freevars=(), cellvars=(), /)\n"
" firstlineno, linetable, freevars=(), cellvars=(), /)\n"
"--\n"
"\n"
"Create a code object. Not for the faint of heart.");
@ -15,7 +15,7 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
int kwonlyargcount, int nlocals, int stacksize, int flags,
PyObject *code, PyObject *consts, PyObject *names,
PyObject *varnames, PyObject *filename, PyObject *name,
int firstlineno, PyObject *lnotab, PyObject *freevars,
int firstlineno, PyObject *linetable, PyObject *freevars,
PyObject *cellvars);
static PyObject *
@ -35,7 +35,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
PyObject *filename;
PyObject *name;
int firstlineno;
PyObject *lnotab;
PyObject *linetable;
PyObject *freevars = NULL;
PyObject *cellvars = NULL;
@ -114,7 +114,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
_PyArg_BadArgument("code", "argument 14", "bytes", PyTuple_GET_ITEM(args, 13));
goto exit;
}
lnotab = PyTuple_GET_ITEM(args, 13);
linetable = PyTuple_GET_ITEM(args, 13);
if (PyTuple_GET_SIZE(args) < 15) {
goto skip_optional;
}
@ -132,7 +132,7 @@ code_new(PyTypeObject *type, PyObject *args, PyObject *kwargs)
}
cellvars = PyTuple_GET_ITEM(args, 15);
skip_optional:
return_value = code_new_impl(type, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars);
return_value = code_new_impl(type, argcount, posonlyargcount, kwonlyargcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, linetable, freevars, cellvars);
exit:
return return_value;
@ -144,7 +144,7 @@ PyDoc_STRVAR(code_replace__doc__,
" co_flags=-1, co_firstlineno=-1, co_code=None, co_consts=None,\n"
" co_names=None, co_varnames=None, co_freevars=None,\n"
" co_cellvars=None, co_filename=None, co_name=None,\n"
" co_lnotab=None)\n"
" co_linetable=None)\n"
"--\n"
"\n"
"Return a copy of the code object with new values for the specified fields.");
@ -160,13 +160,13 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
PyObject *co_consts, PyObject *co_names,
PyObject *co_varnames, PyObject *co_freevars,
PyObject *co_cellvars, PyObject *co_filename,
PyObject *co_name, PyBytesObject *co_lnotab);
PyObject *co_name, PyBytesObject *co_linetable);
static PyObject *
code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
{
PyObject *return_value = NULL;
static const char * const _keywords[] = {"co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", "co_firstlineno", "co_code", "co_consts", "co_names", "co_varnames", "co_freevars", "co_cellvars", "co_filename", "co_name", "co_lnotab", NULL};
static const char * const _keywords[] = {"co_argcount", "co_posonlyargcount", "co_kwonlyargcount", "co_nlocals", "co_stacksize", "co_flags", "co_firstlineno", "co_code", "co_consts", "co_names", "co_varnames", "co_freevars", "co_cellvars", "co_filename", "co_name", "co_linetable", NULL};
static _PyArg_Parser _parser = {NULL, _keywords, "replace", 0};
PyObject *argsbuf[16];
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
@ -185,7 +185,7 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
PyObject *co_cellvars = self->co_cellvars;
PyObject *co_filename = self->co_filename;
PyObject *co_name = self->co_name;
PyBytesObject *co_lnotab = (PyBytesObject *)self->co_lnotab;
PyBytesObject *co_linetable = (PyBytesObject *)self->co_linetable;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 0, 0, argsbuf);
if (!args) {
@ -344,14 +344,14 @@ code_replace(PyCodeObject *self, PyObject *const *args, Py_ssize_t nargs, PyObje
}
}
if (!PyBytes_Check(args[15])) {
_PyArg_BadArgument("replace", "argument 'co_lnotab'", "bytes", args[15]);
_PyArg_BadArgument("replace", "argument 'co_linetable'", "bytes", args[15]);
goto exit;
}
co_lnotab = (PyBytesObject *)args[15];
co_linetable = (PyBytesObject *)args[15];
skip_optional_kwonly:
return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_lnotab);
return_value = code_replace_impl(self, co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals, co_stacksize, co_flags, co_firstlineno, co_code, co_consts, co_names, co_varnames, co_freevars, co_cellvars, co_filename, co_name, co_linetable);
exit:
return return_value;
}
/*[clinic end generated code: output=18c31941ec09e9ca input=a9049054013a1b77]*/
/*[clinic end generated code: output=e3091c7baaaaa420 input=a9049054013a1b77]*/

386
Objects/codeobject.c
View File

@ -119,7 +119,7 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
PyObject *code, PyObject *consts, PyObject *names,
PyObject *varnames, PyObject *freevars, PyObject *cellvars,
PyObject *filename, PyObject *name, int firstlineno,
PyObject *lnotab)
PyObject *linetable)
{
PyCodeObject *co;
Py_ssize_t *cell2arg = NULL;
@ -137,7 +137,7 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
cellvars == NULL || !PyTuple_Check(cellvars) ||
name == NULL || !PyUnicode_Check(name) ||
filename == NULL || !PyUnicode_Check(filename) ||
lnotab == NULL || !PyBytes_Check(lnotab)) {
linetable == NULL || !PyBytes_Check(linetable)) {
PyErr_BadInternalCall();
return NULL;
}
@ -258,8 +258,8 @@ PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount,
Py_INCREF(name);
co->co_name = name;
co->co_firstlineno = firstlineno;
Py_INCREF(lnotab);
co->co_lnotab = lnotab;
Py_INCREF(linetable);
co->co_linetable = linetable;
co->co_zombieframe = NULL;
co->co_weakreflist = NULL;
co->co_extra = NULL;
@ -277,12 +277,12 @@ PyCode_New(int argcount, int kwonlyargcount,
PyObject *code, PyObject *consts, PyObject *names,
PyObject *varnames, PyObject *freevars, PyObject *cellvars,
PyObject *filename, PyObject *name, int firstlineno,
PyObject *lnotab)
PyObject *linetable)
{
return PyCode_NewWithPosOnlyArgs(argcount, 0, kwonlyargcount, nlocals,
stacksize, flags, code, consts, names,
varnames, freevars, cellvars, filename,
name, firstlineno, lnotab);
name, firstlineno, linetable);
}
int
@ -369,7 +369,7 @@ PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
filename_ob, /* filename */
funcname_ob, /* name */
firstlineno, /* firstlineno */
emptystring /* lnotab */
emptystring /* linetable */
);
failed:
@ -395,11 +395,89 @@ static PyMemberDef code_memberlist[] = {
{"co_cellvars", T_OBJECT, OFF(co_cellvars), READONLY},
{"co_filename", T_OBJECT, OFF(co_filename), READONLY},
{"co_name", T_OBJECT, OFF(co_name), READONLY},
{"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
{"co_lnotab", T_OBJECT, OFF(co_lnotab), READONLY},
{"co_firstlineno", T_INT, OFF(co_firstlineno), READONLY},
{"co_linetable", T_OBJECT, OFF(co_linetable), READONLY},
{NULL} /* Sentinel */
};
static int
emit_pair(PyObject **bytes, int *offset, int a, int b)
{
Py_ssize_t len = PyBytes_GET_SIZE(*bytes);
if (*offset + 2 >= len) {
if (_PyBytes_Resize(bytes, len * 2) < 0)
return 0;
}
unsigned char *lnotab = (unsigned char *)
PyBytes_AS_STRING(*bytes) + *offset;
*lnotab++ = a;
*lnotab++ = b;
*offset += 2;
return 1;
}
static int
emit_delta(PyObject **bytes, int bdelta, int ldelta, int *offset)
{
while (bdelta > 255) {
if (!emit_pair(bytes, offset, 255, 0)) {
return 0;
}
bdelta -= 255;
}
while (ldelta > 127) {
if (!emit_pair(bytes, offset, bdelta, 127)) {
return 0;
}
bdelta = 0;
ldelta -= 127;
}
while (ldelta < -128) {
if (!emit_pair(bytes, offset, bdelta, -128)) {
return 0;
}
bdelta = 0;
ldelta += 128;
}
return emit_pair(bytes, offset, bdelta, ldelta);
}
static PyObject *
code_getlnotab(PyCodeObject *code, void *closure)
{
PyCodeAddressRange bounds;
PyObject *bytes;
int table_offset = 0;
int code_offset = 0;
int line = code->co_firstlineno;
bytes = PyBytes_FromStringAndSize(NULL, 64);
if (bytes == NULL) {
return NULL;
}
_PyCode_InitAddressRange(code, &bounds);
while (PyLineTable_NextAddressRange(&bounds)) {
if (bounds.ar_computed_line != line) {
int bdelta = bounds.ar_start - code_offset;
int ldelta = bounds.ar_computed_line - line;
if (!emit_delta(&bytes, bdelta, ldelta, &table_offset)) {
Py_DECREF(bytes);
return NULL;
}
code_offset = bounds.ar_start;
line = bounds.ar_computed_line;
}
}
_PyBytes_Resize(&bytes, table_offset);
return bytes;
}
static PyGetSetDef code_getsetlist[] = {
{"co_lnotab", (getter)code_getlnotab, NULL, NULL},
{0}
};
/* Helper for code_new: return a shallow copy of a tuple that is
guaranteed to contain exact strings, by converting string subclasses
to exact strings and complaining if a non-string is found. */
@ -459,7 +537,7 @@ code.__new__ as code_new
filename: unicode
name: unicode
firstlineno: int
lnotab: object(subclass_of="&PyBytes_Type")
linetable: object(subclass_of="&PyBytes_Type")
freevars: object(subclass_of="&PyTuple_Type", c_default="NULL") = ()
cellvars: object(subclass_of="&PyTuple_Type", c_default="NULL") = ()
/
@ -472,9 +550,9 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
int kwonlyargcount, int nlocals, int stacksize, int flags,
PyObject *code, PyObject *consts, PyObject *names,
PyObject *varnames, PyObject *filename, PyObject *name,
int firstlineno, PyObject *lnotab, PyObject *freevars,
int firstlineno, PyObject *linetable, PyObject *freevars,
PyObject *cellvars)
/*[clinic end generated code: output=612aac5395830184 input=85e678ea4178f234]*/
/*[clinic end generated code: output=42c1839b082ba293 input=0ec80da632b99f57]*/
{
PyObject *co = NULL;
PyObject *ournames = NULL;
@ -540,7 +618,7 @@ code_new_impl(PyTypeObject *type, int argcount, int posonlyargcount,
code, consts, ournames,
ourvarnames, ourfreevars,
ourcellvars, filename,
name, firstlineno, lnotab);
name, firstlineno, linetable);
cleanup:
Py_XDECREF(ournames);
Py_XDECREF(ourvarnames);
@ -584,7 +662,7 @@ code_dealloc(PyCodeObject *co)
Py_XDECREF(co->co_cellvars);
Py_XDECREF(co->co_filename);
Py_XDECREF(co->co_name);
Py_XDECREF(co->co_lnotab);
Py_XDECREF(co->co_linetable);
if (co->co_cell2arg != NULL)
PyMem_FREE(co->co_cell2arg);
if (co->co_zombieframe != NULL)
@ -636,7 +714,7 @@ code.replace
co_cellvars: object(subclass_of="&PyTuple_Type", c_default="self->co_cellvars") = None
co_filename: unicode(c_default="self->co_filename") = None
co_name: unicode(c_default="self->co_name") = None
co_lnotab: PyBytesObject(c_default="(PyBytesObject *)self->co_lnotab") = None
co_linetable: PyBytesObject(c_default="(PyBytesObject *)self->co_linetable") = None
Return a copy of the code object with new values for the specified fields.
[clinic start generated code]*/
@ -649,8 +727,8 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
PyObject *co_consts, PyObject *co_names,
PyObject *co_varnames, PyObject *co_freevars,
PyObject *co_cellvars, PyObject *co_filename,
PyObject *co_name, PyBytesObject *co_lnotab)
/*[clinic end generated code: output=25c8e303913bcace input=d9051bc8f24e6b28]*/
PyObject *co_name, PyBytesObject *co_linetable)
/*[clinic end generated code: output=50d77e668d3b449b input=a5f997b173d7f636]*/
{
#define CHECK_INT_ARG(ARG) \
if (ARG < 0) { \
@ -680,7 +758,7 @@ code_replace_impl(PyCodeObject *self, int co_argcount,
co_argcount, co_posonlyargcount, co_kwonlyargcount, co_nlocals,
co_stacksize, co_flags, (PyObject*)co_code, co_consts, co_names,
co_varnames, co_freevars, co_cellvars, co_filename, co_name,
co_firstlineno, (PyObject*)co_lnotab);
co_firstlineno, (PyObject*)co_linetable);
}
static PyObject *
@ -933,10 +1011,189 @@ code_hash(PyCodeObject *co)
return h;
}
typedef struct {
PyObject_HEAD
PyCodeObject *li_code;
PyCodeAddressRange li_line;
char *li_end;
} lineiterator;
static void
lineiter_dealloc(lineiterator *li)
{
Py_DECREF(li->li_code);
Py_TYPE(li)->tp_free(li);
}
static PyObject *
lineiter_next(lineiterator *li)
{
PyCodeAddressRange *bounds = &li->li_line;
if (!PyLineTable_NextAddressRange(bounds)) {
return NULL;
}
PyObject *start = NULL;
PyObject *end = NULL;
PyObject *line = NULL;
PyObject *result = PyTuple_New(3);
start = PyLong_FromLong(bounds->ar_start);
end = PyLong_FromLong(bounds->ar_end);
if (bounds->ar_line < 0) {
Py_INCREF(Py_None);
line = Py_None;
}
else {
line = PyLong_FromLong(bounds->ar_line);
}
if (result == NULL || start == NULL || end == NULL || line == NULL) {
goto error;
}
PyTuple_SET_ITEM(result, 0, start);
PyTuple_SET_ITEM(result, 1, end);
PyTuple_SET_ITEM(result, 2, line);
return result;
error:
Py_XDECREF(start);
Py_XDECREF(end);
Py_XDECREF(line);
Py_XDECREF(result);
return result;
}
static PyTypeObject LineIterator = {
PyVarObject_HEAD_INIT(&PyType_Type, 0)
"line_iterator", /* tp_name */
sizeof(lineiterator), /* tp_basicsize */
0, /* tp_itemsize */
/* methods */
(destructor)lineiter_dealloc, /* tp_dealloc */
0, /* tp_vectorcall_offset */
0, /* tp_getattr */
0, /* tp_setattr */
0, /* tp_as_async */
0, /* tp_repr */
0, /* tp_as_number */
0, /* tp_as_sequence */
0, /* tp_as_mapping */
0, /* tp_hash */
0, /* tp_call */
0, /* tp_str */
0, /* tp_getattro */
0, /* tp_setattro */
0, /* tp_as_buffer */
Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /* tp_flags */
0, /* tp_doc */
0, /* tp_traverse */
0, /* tp_clear */
0, /* tp_richcompare */
0, /* tp_weaklistoffset */
PyObject_SelfIter, /* tp_iter */
(iternextfunc)lineiter_next, /* tp_iternext */
0, /* tp_methods */
0, /* tp_members */
0, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
0, /* tp_descr_set */
0, /* tp_dictoffset */
0, /* tp_init */
0, /* tp_alloc */
0, /* tp_new */
PyObject_Del, /* tp_free */
};
static PyObject *
code_linesiterator(PyCodeObject *code, PyObject *Py_UNUSED(args))
{
lineiterator *li = (lineiterator *)PyType_GenericAlloc(&LineIterator, 0);
if (li == NULL) {
return NULL;
}
Py_INCREF(code);
li->li_code = code;
_PyCode_InitAddressRange(code, &li->li_line);
return (PyObject *)li;
}
static void
retreat(PyCodeAddressRange *bounds)
{
int ldelta = ((signed char *)bounds->lo_next)[-1];
if (ldelta == -128) {
ldelta = 0;
}
bounds->ar_computed_line -= ldelta;
bounds->lo_next -= 2;
bounds->ar_end = bounds->ar_start;
bounds->ar_start -= ((unsigned char *)bounds->lo_next)[-2];
ldelta = ((signed char *)bounds->lo_next)[-1];
if (ldelta == -128) {
bounds->ar_line = -1;
}
else {
bounds->ar_line = bounds->ar_computed_line;
}
}
static void
advance(PyCodeAddressRange *bounds)
{
bounds->ar_start = bounds->ar_end;
int delta = ((unsigned char *)bounds->lo_next)[0];
assert (delta < 255);
bounds->ar_end += delta;
int ldelta = ((signed char *)bounds->lo_next)[1];
bounds->lo_next += 2;
if (ldelta == -128) {
bounds->ar_line = -1;
}
else {
bounds->ar_computed_line += ldelta;
bounds->ar_line = bounds->ar_computed_line;
}
}
static inline int
at_end(PyCodeAddressRange *bounds) {
return ((unsigned char *)bounds->lo_next)[0] == 255;
}
int
PyLineTable_PreviousAddressRange(PyCodeAddressRange *range)
{
if (range->ar_start <= 0) {
return 0;
}
retreat(range);
while (range->ar_start == range->ar_end) {
assert(range->ar_start > 0);
retreat(range);
}
return 1;
}
int
PyLineTable_NextAddressRange(PyCodeAddressRange *range)
{
if (at_end(range)) {
return 0;
}
advance(range);
while (range->ar_start == range->ar_end) {
assert(!at_end(range));
advance(range);
}
return 1;
}
/* XXX code objects need to participate in GC? */
static struct PyMethodDef code_methods[] = {
{"__sizeof__", (PyCFunction)code_sizeof, METH_NOARGS},
{"co_lines", (PyCFunction)code_linesiterator, METH_NOARGS},
CODE_REPLACE_METHODDEF
{NULL, NULL} /* sentinel */
};
@ -971,7 +1228,7 @@ PyTypeObject PyCode_Type = {
0, /* tp_iternext */
code_methods, /* tp_methods */
code_memberlist, /* tp_members */
0, /* tp_getset */
code_getsetlist, /* tp_getset */
0, /* tp_base */
0, /* tp_dict */
0, /* tp_descr_get */
@ -982,78 +1239,55 @@ PyTypeObject PyCode_Type = {
code_new, /* tp_new */
};
/* Use co_lnotab to compute the line number from a bytecode index, addrq. See
/* Use co_linetable to compute the line number from a bytecode index, addrq. See
lnotab_notes.txt for the details of the lnotab representation.
*/
int
PyCode_Addr2Line(PyCodeObject *co, int addrq)
{
Py_ssize_t size = PyBytes_Size(co->co_lnotab) / 2;
unsigned char *p = (unsigned char*)PyBytes_AsString(co->co_lnotab);
int line = co->co_firstlineno;
int addr = 0;
while (--size >= 0) {
addr += *p++;
if (addr > addrq)
break;
line += (signed char)*p;
p++;
if (addrq == -1) {
return co->co_firstlineno;
}
return line;
assert(addrq >= 0 && addrq < PyBytes_GET_SIZE(co->co_code));
PyCodeAddressRange bounds;
_PyCode_InitAddressRange(co, &bounds);
return _PyCode_CheckLineNumber(addrq, &bounds);
}
void
PyLineTable_InitAddressRange(char *linetable, int firstlineno, PyCodeAddressRange *range)
{
range->lo_next = linetable;
range->ar_start = -1;
range->ar_end = 0;
range->ar_computed_line = range->ar_line = firstlineno;
}
int
_PyCode_InitAddressRange(PyCodeObject* co, PyCodeAddressRange *bounds)
{
char *linetable = PyBytes_AS_STRING(co->co_linetable);
PyLineTable_InitAddressRange(linetable, co->co_firstlineno, bounds);
return bounds->ar_line;
}
/* Update *bounds to describe the first and one-past-the-last instructions in
the same line as lasti. Return the number of that line. */
the same line as lasti. Return the number of that line, or -1 if lasti is out of bounds. */
int
_PyCode_CheckLineNumber(PyCodeObject* co, int lasti, PyAddrPair *bounds)
_PyCode_CheckLineNumber(int lasti, PyCodeAddressRange *bounds)
{
Py_ssize_t size;
int addr, line;
unsigned char* p;
p = (unsigned char*)PyBytes_AS_STRING(co->co_lnotab);
size = PyBytes_GET_SIZE(co->co_lnotab) / 2;
addr = 0;
line = co->co_firstlineno;
assert(line > 0);
/* possible optimization: if f->f_lasti == instr_ub
(likely to be a common case) then we already know
instr_lb -- if we stored the matching value of p
somewhere we could skip the first while loop. */
/* See lnotab_notes.txt for the description of
co_lnotab. A point to remember: increments to p
come in (addr, line) pairs. */
bounds->ap_lower = 0;
while (size > 0) {
if (addr + *p > lasti)
break;
addr += *p++;
if ((signed char)*p)
bounds->ap_lower = addr;
line += (signed char)*p;
p++;
--size;
}
if (size > 0) {
while (--size >= 0) {
addr += *p++;
if ((signed char)*p)
break;
p++;
while (bounds->ar_end <= lasti) {
if (!PyLineTable_NextAddressRange(bounds)) {
return -1;
}
bounds->ap_upper = addr;
}
else {
bounds->ap_upper = INT_MAX;
while (bounds->ar_start > lasti) {
if (!PyLineTable_PreviousAddressRange(bounds)) {
return -1;
}
}
return line;
return bounds->ar_line;
}

36
Objects/frameobject.c
View File

@ -249,36 +249,22 @@ explain_incompatible_block_stack(int64_t to_stack)
static int *
marklines(PyCodeObject *code, int len)
{
PyCodeAddressRange bounds;
_PyCode_InitAddressRange(code, &bounds);
assert (bounds.ar_end == 0);
int *linestarts = PyMem_New(int, len);
if (linestarts == NULL) {
return NULL;
}
Py_ssize_t size = PyBytes_GET_SIZE(code->co_lnotab) / 2;
unsigned char *p = (unsigned char*)PyBytes_AS_STRING(code->co_lnotab);
int line = code->co_firstlineno;
int addr = 0;
int index = 0;
while (--size >= 0) {
addr += *p++;
if (index*2 < addr) {
linestarts[index++] = line;
}
while (index*2 < addr) {
linestarts[index++] = -1;
if (index >= len) {
break;
}
}
line += (signed char)*p;
p++;
for (int i = 0; i < len; i++) {
linestarts[i] = -1;
}
if (index < len) {
linestarts[index++] = line;
while (PyLineTable_NextAddressRange(&bounds)) {
assert(bounds.ar_start/2 < len);
linestarts[bounds.ar_start/2] = bounds.ar_line;
}
while (index < len) {
linestarts[index++] = -1;
}
assert(index == len);
return linestarts;
}
@ -925,7 +911,7 @@ _PyFrame_New_NoTrack(PyThreadState *tstate, PyCodeObject *code,
}
f->f_lasti = -1;
f->f_lineno = code->co_firstlineno;
f->f_lineno = 0;
f->f_iblock = 0;
f->f_state = FRAME_CREATED;
f->f_gen = NULL;

106
Objects/lnotab_notes.txt
View File

@ -1,11 +1,103 @@
All about co_lnotab, the line number table.
Description of the internal format of the line number table
Conceptually, the line number table consists of a sequence of triples:
start-offset (inclusive), end-offset (exclusive), line-number.
Note that note all byte codes have a line number so we need handle `None` for the line-number.
However, storing the above sequence directly would be very inefficient as we would need 12 bytes per entry.
First of all, we can note that the end of one entry is the same as the start of the next, so we can overlap entries.
Secondly we also note that we don't really need arbitrary access to the sequence, so we can store deltas.
We just need to store (end - start, line delta) pairs. The start offset of the first entry is always zero.
Thirdly, most deltas are small, so we can use a single byte for each value, as long we allow several entries for the same line.
Consider the following table
Start End Line
0 6 1
6 50 2
50 350 7
350 360 No line number
360 376 8
376 380 208
Stripping the redundant ends gives:
End-Start Line-delta
6 +1
44 +1
300 +5
10 No line number
16 +1
4 +200
Note that the end - start value is always positive.
Finally in order, to fit into a single byte we need to convert start deltas to the range 0 <= delta <= 254,
and line deltas to the range -127 <= delta <= 127.
A line delta of -128 is used to indicate no line number.
A start delta of 255 is used as a sentinel to mark the end of the table.
Also note that a delta of zero indicates that there are no bytecodes in the given range,
which means can use an invalidate line number for that range.
Final form:
Start delta Line delta
6 +1
44 +1
254 +5
46 0
10 -128 (No line number, treated as a delta of zero)
16 +1
0 +127 (line 135, but the range is empty as no bytecodes are at line 135)
4 +73
255 (end mark) ---
Iterating over the table.
-------------------------
For the `co_lines` attribute we want to emit the full form, omitting the (350, 360, No line number) and empty entries.
The code is as follows:
def co_lines(code):
line = code.co_firstlineno
end = 0
table_iter = iter(code.internal_line_table):
for sdelta, ldelta in table_iter:
if sdelta == 255:
break
if ldelta == 0: # No change to line number, just accumulate changes to end
end += odelta
continue
start = end
end = start + sdelta
if ldelta == -128: # No valid line number -- skip entry
continue
line += ldelta
if end == start: # Empty range, omit.
continue
yield start, end, line
The historical co_lnotab format
-------------------------------
prior to 3.10 code objects stored a field named co_lnotab.
This was an array of unsigned bytes disguised as a Python bytes object.
The old co_lnotab did not account for the presence of bytecodes without a line number,
nor was it well suited to tracing as a number of workarounds were required.
The old format can still be accessed via `code.co_lnotab`, which is lazily computed from the new format.
Below is the description of the old co_lnotab format:
Code objects store a field named co_lnotab. This is an array of unsigned bytes
disguised as a Python bytes object. It is used to map bytecode offsets to
source code line #s for tracebacks and to identify line number boundaries for
line tracing. Because of internals of the peephole optimizer, it's possible
for lnotab to contain bytecode offsets that are no longer valid (for example
if the optimizer removed the last line in a function).
The array is conceptually a compressed list of
(bytecode offset increment, line number increment)

73
Python/ceval.c
View File

@ -66,8 +66,8 @@ static void call_exc_trace(Py_tracefunc, PyObject *,
PyThreadState *, PyFrameObject *);
static int maybe_call_line_trace(Py_tracefunc, PyObject *,
PyThreadState *, PyFrameObject *,
int *, int *, int *);
static void maybe_dtrace_line(PyFrameObject *, int *, int *, int *);
PyCodeAddressRange *, int *);
static void maybe_dtrace_line(PyFrameObject *, PyCodeAddressRange *, int *);
static void dtrace_function_entry(PyFrameObject *);
static void dtrace_function_return(PyFrameObject *);
@ -976,7 +976,6 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
is true when the line being executed has changed. The
initial values are such as to make this false the first
time it is tested. */
int instr_ub = -1, instr_lb = 0, instr_prev = -1;
const _Py_CODEUNIT *first_instr;
PyObject *names;
@ -1390,6 +1389,10 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, PyFrameObject *f, int throwflag)
dtrace_function_entry(f);
co = f->f_code;
PyCodeAddressRange bounds;
_PyCode_InitAddressRange(co, &bounds);
int instr_prev = -1;
names = co->co_names;
consts = co->co_consts;
fastlocals = f->f_localsplus;
@ -1514,7 +1517,7 @@ main_loop:
f->f_lasti = INSTR_OFFSET();
if (PyDTrace_LINE_ENABLED())
maybe_dtrace_line(f, &instr_lb, &instr_ub, &instr_prev);
maybe_dtrace_line(f, &bounds, &instr_prev);
/* line-by-line tracing support */
@ -1528,7 +1531,7 @@ main_loop:
err = maybe_call_line_trace(tstate->c_tracefunc,
tstate->c_traceobj,
tstate, f,
&instr_lb, &instr_ub, &instr_prev);
&bounds, &instr_prev);
/* Reload possibly changed frame fields */
JUMPTO(f->f_lasti);
stack_pointer = f->f_valuestack+f->f_stackdepth;
@ -4039,14 +4042,7 @@ exception_unwind:
PUSH(exc);
JUMPTO(handler);
if (_Py_TracingPossible(ceval2)) {
int needs_new_execution_window = (f->f_lasti < instr_lb || f->f_lasti >= instr_ub);
int needs_line_update = (f->f_lasti == instr_lb || f->f_lasti < instr_prev);
/* Make sure that we trace line after exception if we are in a new execution
* window or we don't need a line update and we are not in the first instruction
* of the line. */
if (needs_new_execution_window || (!needs_line_update && instr_lb > 0)) {
instr_prev = INT_MAX;
}
instr_prev = INT_MAX;
}
/* Resume normal execution */
f->f_state = FRAME_EXECUTING;
@ -4960,7 +4956,7 @@ _PyEval_CallTracing(PyObject *func, PyObject *args)
static int
maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
PyThreadState *tstate, PyFrameObject *frame,
int *instr_lb, int *instr_ub, int *instr_prev)
PyCodeAddressRange *bounds, int *instr_prev)
{
int result = 0;
int line = frame->f_lineno;
@ -4968,21 +4964,17 @@ maybe_call_line_trace(Py_tracefunc func, PyObject *obj,
/* If the last instruction executed isn't in the current
instruction window, reset the window.
*/
if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) {
PyAddrPair bounds;
line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti,
&bounds);
*instr_lb = bounds.ap_lower;
*instr_ub = bounds.ap_upper;
}
line = _PyCode_CheckLineNumber(frame->f_lasti, bounds);
/* If the last instruction falls at the start of a line or if it
represents a jump backwards, update the frame's line number and
then call the trace function if we're tracing source lines.
*/
if ((frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev)) {
frame->f_lineno = line;
if (frame->f_trace_lines) {
result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
if ((line != frame->f_lineno || frame->f_lasti < *instr_prev)) {
if (line != -1) {
frame->f_lineno = line;
if (frame->f_trace_lines) {
result = call_trace(func, obj, tstate, frame, PyTrace_LINE, Py_None);
}
}
}
/* Always emit an opcode event if we're tracing all opcodes. */
@ -5907,33 +5899,28 @@ dtrace_function_return(PyFrameObject *f)
/* DTrace equivalent of maybe_call_line_trace. */
static void
maybe_dtrace_line(PyFrameObject *frame,
int *instr_lb, int *instr_ub, int *instr_prev)
PyCodeAddressRange *bounds, int *instr_prev)
{
int line = frame->f_lineno;
const char *co_filename, *co_name;
/* If the last instruction executed isn't in the current
instruction window, reset the window.
*/
if (frame->f_lasti < *instr_lb || frame->f_lasti >= *instr_ub) {
PyAddrPair bounds;
line = _PyCode_CheckLineNumber(frame->f_code, frame->f_lasti,
&bounds);
*instr_lb = bounds.ap_lower;
*instr_ub = bounds.ap_upper;
}
int line = _PyCode_CheckLineNumber(frame->f_lasti, bounds);
/* If the last instruction falls at the start of a line or if
it represents a jump backwards, update the frame's line
number and call the trace function. */
if (frame->f_lasti == *instr_lb || frame->f_lasti < *instr_prev) {
frame->f_lineno = line;
co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename);
if (!co_filename)
co_filename = "?";
co_name = PyUnicode_AsUTF8(frame->f_code->co_name);
if (!co_name)
co_name = "?";
PyDTrace_LINE(co_filename, co_name, line);
if (line != frame->f_lineno || frame->f_lasti < *instr_prev) {
if (line != -1) {
frame->f_lineno = line;
co_filename = PyUnicode_AsUTF8(frame->f_code->co_filename);
if (!co_filename)
co_filename = "?";
co_name = PyUnicode_AsUTF8(frame->f_code->co_name);
if (!co_name)
co_name = "?";
PyDTrace_LINE(co_filename, co_name, line);
}
}
*instr_prev = frame->f_lasti;
}

220
Python/compile.c
View File

@ -1827,7 +1827,7 @@ compiler_mod(struct compiler *c, mod_ty mod)
return NULL;
}
/* Use 0 for firstlineno initially, will fixup in assemble(). */
if (!compiler_enter_scope(c, module, COMPILER_SCOPE_MODULE, mod, 0))
if (!compiler_enter_scope(c, module, COMPILER_SCOPE_MODULE, mod, 1))
return NULL;
switch (mod->kind) {
case Module_kind:
@ -2271,7 +2271,9 @@ compiler_function(struct compiler *c, stmt_ty s, int is_async)
c->u->u_argcount = asdl_seq_LEN(args->args);
c->u->u_posonlyargcount = asdl_seq_LEN(args->posonlyargs);
c->u->u_kwonlyargcount = asdl_seq_LEN(args->kwonlyargs);
VISIT_SEQ_IN_SCOPE(c, stmt, body);
for (i = docstring ? 1 : 0; i < asdl_seq_LEN(body); i++) {
VISIT(c, stmt, (stmt_ty)asdl_seq_GET(body, i));
}
co = assemble(c, 1);
qualname = c->u->u_qualname;
Py_INCREF(qualname);
@ -2808,6 +2810,8 @@ compiler_async_for(struct compiler *c, stmt_ty s)
/* Except block for __anext__ */
compiler_use_next_block(c, except);
c->u->u_lineno = -1;
ADDOP(c, END_ASYNC_FOR);
/* `else` block */
@ -3115,7 +3119,8 @@ compiler_try_except(struct compiler *c, stmt_ty s)
compiler_pop_fblock(c, HANDLER_CLEANUP, cleanup_body);
ADDOP(c, POP_BLOCK);
ADDOP(c, POP_EXCEPT);
/* name = None; del name */
/* name = None; del name; # Mark as artificial */
c->u->u_lineno = -1;
ADDOP_LOAD_CONST(c, Py_None);
compiler_nameop(c, handler->v.ExceptHandler.name, Store);
compiler_nameop(c, handler->v.ExceptHandler.name, Del);
@ -3124,7 +3129,8 @@ compiler_try_except(struct compiler *c, stmt_ty s)
/* except: */
compiler_use_next_block(c, cleanup_end);
/* name = None; del name */
/* name = None; del name; # Mark as artificial */
c->u->u_lineno = -1;
ADDOP_LOAD_CONST(c, Py_None);
compiler_nameop(c, handler->v.ExceptHandler.name, Store);
compiler_nameop(c, handler->v.ExceptHandler.name, Del);
@ -3359,6 +3365,7 @@ compiler_visit_stmt_expr(struct compiler *c, expr_ty value)
if (value->kind == Constant_kind) {
/* ignore constant statement */
ADDOP(c, NOP);
return 1;
}
@ -3431,6 +3438,7 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
case Expr_kind:
return compiler_visit_stmt_expr(c, s->v.Expr.value);
case Pass_kind:
ADDOP(c, NOP);
break;
case Break_kind:
return compiler_break(c);
@ -5429,8 +5437,9 @@ struct assembler {
basicblock **a_reverse_postorder; /* list of blocks in dfs postorder */
PyObject *a_lnotab; /* string containing lnotab */
int a_lnotab_off; /* offset into lnotab */
int a_lineno; /* last lineno of emitted instruction */
int a_lineno_off; /* bytecode offset of last lineno */
int a_prevlineno; /* lineno of last emitted line in line table */
int a_lineno; /* lineno of last emitted instruction */
int a_lineno_start; /* bytecode start offset of current lineno */
};
static void
@ -5533,7 +5542,7 @@ static int
assemble_init(struct assembler *a, int nblocks, int firstlineno)
{
memset(a, 0, sizeof(struct assembler));
a->a_lineno = firstlineno;
a->a_prevlineno = a->a_lineno = firstlineno;
a->a_bytecode = PyBytes_FromStringAndSize(NULL, DEFAULT_CODE_SIZE);
if (!a->a_bytecode)
return 0;
@ -5573,114 +5582,82 @@ blocksize(basicblock *b)
return size;
}
/* Appends a pair to the end of the line number table, a_lnotab, representing
the instruction's bytecode offset and line number. See
Objects/lnotab_notes.txt for the description of the line number table. */
static int
assemble_lnotab(struct assembler *a, struct instr *i)
assemble_emit_linetable_pair(struct assembler *a, int bdelta, int ldelta)
{
int d_bytecode, d_lineno;
Py_ssize_t len;
unsigned char *lnotab;
d_lineno = i->i_lineno - a->a_lineno;
if (d_lineno == 0) {
return 1;
}
d_bytecode = (a->a_offset - a->a_lineno_off) * sizeof(_Py_CODEUNIT);
assert(d_bytecode >= 0);
if (d_bytecode > 255) {
int j, nbytes, ncodes = d_bytecode / 255;
nbytes = a->a_lnotab_off + 2 * ncodes;
len = PyBytes_GET_SIZE(a->a_lnotab);
if (nbytes >= len) {
if ((len <= INT_MAX / 2) && (len * 2 < nbytes))
len = nbytes;
else if (len <= INT_MAX / 2)
len *= 2;
else {
PyErr_NoMemory();
return 0;
}
if (_PyBytes_Resize(&a->a_lnotab, len) < 0)
return 0;
}
lnotab = (unsigned char *)
PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
for (j = 0; j < ncodes; j++) {
*lnotab++ = 255;
*lnotab++ = 0;
}
d_bytecode -= ncodes * 255;
a->a_lnotab_off += ncodes * 2;
}
assert(0 <= d_bytecode && d_bytecode <= 255);
if (d_lineno < -128 || 127 < d_lineno) {
int j, nbytes, ncodes, k;
if (d_lineno < 0) {
k = -128;
/* use division on positive numbers */
ncodes = (-d_lineno) / 128;
}
else {
k = 127;
ncodes = d_lineno / 127;
}
d_lineno -= ncodes * k;
assert(ncodes >= 1);
nbytes = a->a_lnotab_off + 2 * ncodes;
len = PyBytes_GET_SIZE(a->a_lnotab);
if (nbytes >= len) {
if ((len <= INT_MAX / 2) && len * 2 < nbytes)
len = nbytes;
else if (len <= INT_MAX / 2)
len *= 2;
else {
PyErr_NoMemory();
return 0;
}
if (_PyBytes_Resize(&a->a_lnotab, len) < 0)
return 0;
}
lnotab = (unsigned char *)
PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
*lnotab++ = d_bytecode;
*lnotab++ = k;
d_bytecode = 0;
for (j = 1; j < ncodes; j++) {
*lnotab++ = 0;
*lnotab++ = k;
}
a->a_lnotab_off += ncodes * 2;
}
assert(-128 <= d_lineno && d_lineno <= 127);
len = PyBytes_GET_SIZE(a->a_lnotab);
Py_ssize_t len = PyBytes_GET_SIZE(a->a_lnotab);
if (a->a_lnotab_off + 2 >= len) {
if (_PyBytes_Resize(&a->a_lnotab, len * 2) < 0)
return 0;
}
lnotab = (unsigned char *)
unsigned char *lnotab = (unsigned char *)
PyBytes_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
a->a_lnotab_off += 2;
if (d_bytecode) {
*lnotab++ = d_bytecode;
*lnotab++ = d_lineno;
}
else { /* First line of a block; def stmt, etc. */
*lnotab++ = 0;
*lnotab++ = d_lineno;
}
a->a_lineno = i->i_lineno;
a->a_lineno_off = a->a_offset;
*lnotab++ = bdelta;
*lnotab++ = ldelta;
return 1;
}
/* Appends a range to the end of the line number table. See
* Objects/lnotab_notes.txt for the description of the line number table. */
static int
assemble_line_range(struct assembler *a)
{
int ldelta, bdelta;
bdelta = (a->a_offset - a->a_lineno_start) * 2;
if (bdelta == 0) {
return 1;
}
if (a->a_lineno < 0) {
ldelta = -128;
}
else {
ldelta = a->a_lineno - a->a_prevlineno;
a->a_prevlineno = a->a_lineno;
while (ldelta > 127) {
if (!assemble_emit_linetable_pair(a, 0, 127)) {
return 0;
}
ldelta -= 127;
}
while (ldelta < -127) {
if (!assemble_emit_linetable_pair(a, 0, -127)) {
return 0;
}
ldelta += 127;
}
}
assert(-128 <= ldelta && ldelta < 128);
while (bdelta > 254) {
if (!assemble_emit_linetable_pair(a, 254, ldelta)) {
return 0;
}
ldelta = a->a_lineno < 0 ? -128 : 0;
bdelta -= 254;
}
if (!assemble_emit_linetable_pair(a, bdelta, ldelta)) {
return 0;
}
a->a_lineno_start = a->a_offset;
return 1;
}
static int
assemble_lnotab(struct assembler *a, struct instr *i)
{
if (i->i_lineno == a->a_lineno) {
return 1;
}
if (!assemble_line_range(a)) {
return 0;
}
a->a_lineno = i->i_lineno;
return 1;
}
/* assemble_emit()
Extend the bytecode with a new instruction.
Update lnotab if necessary.
@ -5998,7 +5975,7 @@ assemble(struct compiler *c, int addNone)
block ends with a jump or return b_next shouldn't set.
*/
if (!c->u->u_curblock->b_return) {
NEXT_BLOCK(c);
c->u->u_lineno = -1;
if (addNone)
ADDOP_LOAD_CONST(c, Py_None);
ADDOP(c, RETURN_VALUE);
@ -6015,7 +5992,7 @@ assemble(struct compiler *c, int addNone)
if (!c->u->u_firstlineno) {
if (entryblock && entryblock->b_instr && entryblock->b_instr->i_lineno)
c->u->u_firstlineno = entryblock->b_instr->i_lineno;
else
else
c->u->u_firstlineno = 1;
}
if (!assemble_init(&a, nblocks, c->u->u_firstlineno))
@ -6040,6 +6017,13 @@ assemble(struct compiler *c, int addNone)
if (!assemble_emit(&a, &b->b_instr[j]))
goto error;
}
if (!assemble_line_range(&a)) {
return 0;
}
/* Emit sentinel at end of line number table */
if (!assemble_emit_linetable_pair(&a, 255, -128)) {
goto error;
}
if (_PyBytes_Resize(&a.a_lnotab, a.a_lnotab_off) < 0)
goto error;
@ -6280,22 +6264,42 @@ static void
clean_basic_block(basicblock *bb) {
/* Remove NOPs and any code following a return or re-raise. */
int dest = 0;
int prev_lineno = -1;
for (int src = 0; src < bb->b_iused; src++) {
int lineno = bb->b_instr[src].i_lineno;
switch(bb->b_instr[src].i_opcode) {
case NOP:
/* skip */
break;
case RETURN_VALUE:
case RERAISE:
bb->b_next = NULL;
bb->b_instr[dest] = bb->b_instr[src];
dest++;
goto end;
case NOP:
{
/* Eliminate no-op if it doesn't have a line number, or
* if the next instruction has same line number or no line number, or
* if the previous instruction had the same line number. */
if (lineno < 0) {
break;
}
if (prev_lineno == lineno) {
break;
}
if (src < bb->b_iused - 1) {
int next_lineno = bb->b_instr[src+1].i_lineno;
if (next_lineno < 0 || next_lineno == lineno) {
bb->b_instr[src+1].i_lineno = lineno;
break;
}
}
}
/* fallthrough */
default:
if (dest != src) {
bb->b_instr[dest] = bb->b_instr[src];
}
dest++;
prev_lineno = lineno;
break;
}
}

3051
Python/importlib.h generated
View File

File diff suppressed because it is too large Load Diff

4666
Python/importlib_external.h generated
View File

File diff suppressed because it is too large Load Diff

1681
Python/importlib_zipimport.h
View File

File diff suppressed because it is too large Load Diff

12
Python/marshal.c
View File

@ -524,7 +524,7 @@ w_complex_object(PyObject *v, char flag, WFILE *p)
w_object(co->co_filename, p);
w_object(co->co_name, p);
w_long(co->co_firstlineno, p);
w_object(co->co_lnotab, p);
w_object(co->co_linetable, p);
}
else if (PyObject_CheckBuffer(v)) {
/* Write unknown bytes-like objects as a bytes object */
@ -1312,7 +1312,7 @@ r_object(RFILE *p)
PyObject *filename = NULL;
PyObject *name = NULL;
int firstlineno;
PyObject *lnotab = NULL;
PyObject *linetable = NULL;
idx = r_ref_reserve(flag, p);
if (idx < 0)
@ -1367,8 +1367,8 @@ r_object(RFILE *p)
firstlineno = (int)r_long(p);
if (firstlineno == -1 && PyErr_Occurred())
break;
lnotab = r_object(p);
if (lnotab == NULL)
linetable = r_object(p);
if (linetable == NULL)
goto code_error;
if (PySys_Audit("code.__new__", "OOOiiiiii",
@ -1382,7 +1382,7 @@ r_object(RFILE *p)
nlocals, stacksize, flags,
code, consts, names, varnames,
freevars, cellvars, filename, name,
firstlineno, lnotab);
firstlineno, linetable);
v = r_ref_insert(v, idx, flag, p);
code_error:
@ -1394,7 +1394,7 @@ r_object(RFILE *p)
Py_XDECREF(cellvars);
Py_XDECREF(filename);
Py_XDECREF(name);
Py_XDECREF(lnotab);
Py_XDECREF(linetable);
}
retval = v;
break;

17
Tools/gdb/libpython.py
View File

@ -648,19 +648,28 @@ class PyCodeObjectPtr(PyObjectPtr):
Analogous to PyCode_Addr2Line; translated from pseudocode in
Objects/lnotab_notes.txt
'''
co_lnotab = self.pyop_field('co_lnotab').proxyval(set())
co_linetable = self.pyop_field('co_linetable').proxyval(set())
# Initialize lineno to co_firstlineno as per PyCode_Addr2Line
# not 0, as lnotab_notes.txt has it:
lineno = int_from_int(self.field('co_firstlineno'))
if addrq < 0:
return lineno
addr = 0
for addr_incr, line_incr in zip(co_lnotab[::2], co_lnotab[1::2]):
for addr_incr, line_incr in zip(co_linetable[::2], co_linetable[1::2]):
if addr_incr == 255:
break
addr += ord(addr_incr)
line_delta = ord(line_incr)
if line_delta == 128:
line_delta = 0
elif line_delta > 128:
line_delta -= 256
lineno += line_delta
if addr > addrq:
return lineno
lineno += ord(line_incr)
return lineno
assert False, "Unreachable"
class PyDictObjectPtr(PyObjectPtr):